# cvpr25paper9858/ges3vig architecture

> Architecture diagram of the GitHub repository cvpr25paper9858/ges3vig, generated by GitDiagram from its file tree, README and a sample of source files (last updated 2025-05-21). It is an AI-made overview: check details against the source.

- Interactive diagram: https://gitdiagram.com/cvpr25paper9858/ges3vig
- Repository: https://github.com/cvpr25paper9858/ges3vig

## Overview

##### Project type and purpose
- From the README and file tree, **Ges3ViG** is a research‐oriented deep learning codebase for 3D visual grounding that incorporates human pointing gestures.
- It is built on **PyTorch** + **PyTorch Lightning**, uses **Hydra** for config management, and relies on CUDA extensions (MinkowskiEngine, custom C++/CUDA ops) plus external tools (ScanNet v2, ImputeRefer dataset, pre-trained PointGroup detector, WandB for logging).

##### File structure
- Top‐level scripts:
- `train.py`, `test.py`, `evaluate.py` – entry points for training, inference, evaluation
- `setup.py` – package installer
- `Imputer/` – standalone CUDA C++/Python package to generate human‐pose imputations
- Configuration: `config/` holds Hydra YAML files for datasets, models, global settings
- Data pipelines: `dataset/` contains subfolders for ScanNet v2, ScanRefer, ImputeRefer, each with preprocessing & conversion scripts
- Core library: `ges3vig/` package
- `data/` – PyTorch Lightning `DataModule` & datasets
- `model/` – vision, language, cross‐modal modules, the main `Ges3ViG` class
- `loss/`, `evaluation/`, `util/` – supporting components
- `common_ops/` – custom CUDA kernels for clustering, ROI‐pool, etc., compiled via its own `setup.py`

##### README for additional insights
- The README details data preparation workflows, external dependencies (ScanNet, human models, pre-trained detector), and how Hydra config keys map to scripts.

## Diagram (Mermaid)

```mermaid
flowchart TB
    %% Data Sources
    subgraph "Data Sources"
        DS1["ScanNet v2 Meshes & Annotations"]:::data
        DS2["ImputeRefer JSONs"]:::data
        DS3["Human Pose ZIPs"]:::data
        DS4["PointGroup Detector Checkpoint"]:::data
    end

    %% Imputer Module
    subgraph "Imputer Module"
        IMB["Imputer Extension"]:::module
    end

    %% Data Preprocessing
    subgraph "Data Preprocessing"
        PRE1["ScanNet: preprocess_all_data.py"]:::module
        PRE2["ScanNet: preprocess_all_imputed_data.py"]:::module
        PRE3["ScanNet: preprocess_all_imputed_data_add_scans.py"]:::module
        PRE4["ScanNet: imputer_full.py"]:::module
        PRE5["ScanRefer: add_evaluation_labels.py"]:::module
        PRE6["ScanRefer: convert_output_to_benchmark_format.py"]:::module
        PRE7["ImputeRefer: get_referential_data.py"]:::module
        PRE8["ImputeRefer: add_evaluation_labels.py"]:::module
        PRE9["ImputeRefer: postprocess_generation.py"]:::module
        PRE10["ImputeRefer: convert_output_to_benchmark_format.py"]:::module
    end

    %% Configuration Layer
    CONFIG["Config Layer (Hydra)"]:::service

    %% Core Library
    subgraph "Core Library"
        subgraph "DataModule & Datasets"
            DM["DataModule"]:::core
            DSets1["general_dataset.py"]:::core
            DSets2["general_imputed_multi_dataset.py"]:::core
            DSets3["scanrefer.py"]:::core
            DSets4["imputerefer.py"]:::core
            DSets5["pointrefer.py"]:::core
        end
        Model["Ges3ViG Model"]:::core
        subgraph "Vision Module"
            V1["clip_image_encoder.py"]:::core
            V2["human_processor_early_matcher_plus_query_former.py"]:::core
            V3["object_renderer.py"]:::core
            V4["pointgroup.py"]:::core
        end
        subgraph "Language Module"
            L1["clip_word_encoder.py"]:::core
            L2["m3dref_clip.py"]:::core
        end
        subgraph "Cross-Modal Module"
            C1["attention.py"]:::core
            C2["joint_module.py"]:::core
            C3["match_module.py"]:::core
        end
        subgraph "Losses"
            LS1["contrastive_loss.py"]:::core
            LS2["human_losses.py"]:::core
            LS3["pt_offset_loss.py"]:::core
            LS4["reference_loss.py"]:::core
        end
        subgraph "Evaluation"
            E1["scanrefer_evaluator.py"]:::core
            E2["general_evaluator.py"]:::core
            E3["multi3drefer_evaluator.py"]:::core
            E4["referit3d_evaluator.py"]:::core
        end
        Ops["Custom CUDA Ops"]:::core
    end

    %% Training, Inference, Evaluation
    TRAIN["train.py"]:::module
    TEST["test.py"]:::module
    EVAL["evaluate.py"]:::module

    %% External Services
    subgraph "External Services"
        WNB["Weights & Biases"]:::service
        HYDRA["Hydra"]:::service
    end

    %% Data Flows
    DS1 --> PRE1
    DS3 --> IMB
    IMB --> PRE4
    PRE4 --> PRE2
    PRE4 --> PRE3
    DS2 --> PRE7
    PRE1 --> DM
    PRE2 --> DM
    PRE3 --> DM
    PRE5 --> DM
    PRE6 --> DM
    PRE7 --> DM
    PRE8 --> DM
    PRE9 --> DM
    PRE10 --> DM

    CONFIG --> TRAIN
    CONFIG --> TEST
    CONFIG --> EVAL

    TRAIN --> DM
    TRAIN --> Model
    TRAIN --> WNB
    DS4 --> TRAIN

    DM --> Model
    Model --> Ops

    Model --> LS1
    Model --> LS2
    Model --> LS3
    Model --> LS4

    TRAIN --> TEST
    TEST --> DM
    TEST --> EVAL

    EVAL --> E1
    EVAL --> E2
    EVAL --> E3
    EVAL --> E4

    %% Click Events
    click IMB "https://github.com/cvpr25paper9858/ges3vig/blob/master/Imputer/src/imputer.cpp"
    click IMB "https://github.com/cvpr25paper9858/ges3vig/blob/master/Imputer/src/imputer.h"
    click IMB "https://github.com/cvpr25paper9858/ges3vig/blob/master/Imputer/src/imputer_cuda.cu"

    click PRE1 "https://github.com/cvpr25paper9858/ges3vig/blob/master/dataset/scannetv2/preprocess_all_data.py"
    click PRE2 "https://github.com/cvpr25paper9858/ges3vig/blob/master/dataset/scannetv2/preprocess_all_imputed_data.py"
    click PRE3 "https://github.com/cvpr25paper9858/ges3vig/blob/master/dataset/scannetv2/preprocess_all_imputed_data_add_scans.py"
    click PRE4 "https://github.com/cvpr25paper9858/ges3vig/blob/master/dataset/scannetv2/imputer_full.py"

    click PRE5 "https://github.com/cvpr25paper9858/ges3vig/blob/master/dataset/scanrefer/add_evaluation_labels.py"
    click PRE6 "https://github.com/cvpr25paper9858/ges3vig/blob/master/dataset/scanrefer/convert_output_to_benchmark_format.py"

    click PRE7 "https://github.com/cvpr25paper9858/ges3vig/blob/master/dataset/imputerefer/get_referential_data.py"
    click PRE8 "https://github.com/cvpr25paper9858/ges3vig/blob/master/dataset/imputerefer/add_evaluation_labels.py"
    click PRE9 "https://github.com/cvpr25paper9858/ges3vig/blob/master/dataset/imputerefer/postprocess_genereation.py"
    click PRE10 "https://github.com/cvpr25paper9858/ges3vig/blob/master/dataset/imputerefer/convert_output_to_benchmark_format.py"

    click CONFIG "https://github.com/cvpr25paper9858/ges3vig/blob/master/config/global_config.yaml"
    click CONFIG "https://github.com/cvpr25paper9858/ges3vig/blob/master/config/model/ges3vig.yaml"

    click DM "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/data/data_module.py"
    click DSets1 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/data/dataset/general_dataset.py"
    click DSets2 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/data/dataset/general_imputed_multi_dataset.py"
    click DSets3 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/data/dataset/scanrefer.py"
    click DSets4 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/data/dataset/imputerefer.py"
    click DSets5 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/data/dataset/pointrefer.py"

    click Model "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/model/ges3vig.py"
    click V1 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/model/vision_module/clip_image_encoder.py"
    click V2 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/model/vision_module/human_processor_early_matcher_plus_query_former.py"
    click V3 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/model/vision_module/object_renderer.py"
    click V4 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/model/vision_module/pointgroup.py"

    click L1 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/model/language_module/clip_word_encoder.py"
    click L2 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/model/m3dref_clip.py"

    click C1 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/model/cross_modal_module/attention.py"
    click C2 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/model/cross_modal_module/joint_module.py"
    click C3 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/model/cross_modal_module/match_module.py"

    click LS1 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/loss/contrastive_loss.py"
    click LS2 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/loss/human_losses.py"
    click LS3 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/loss/pt_offset_loss.py"
    click LS4 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/loss/reference_loss.py"

    click E1 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/evaluation/scanrefer_evaluator.py"
    click E2 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/evaluation/general_evaluator.py"
    click E3 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/evaluation/multi3drefer_evaluator.py"
    click E4 "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/evaluation/referit3d_evaluator.py"

    click Ops "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/common_ops/functions/common_ops.py"
    click Ops "https://github.com/cvpr25paper9858/ges3vig/blob/master/ges3vig/common_ops/functions/pointgroup_ops.py"

    click TRAIN "https://github.com/cvpr25paper9858/ges3vig/blob/master/train.py"
    click TEST "https://github.com/cvpr25paper9858/ges3vig/blob/master/test.py"
    click EVAL "https://github.com/cvpr25paper9858/ges3vig/blob/master/evaluate.py"

    %% Styles
    classDef data fill:#a8e6a2,stroke:#333,stroke-width:1px
    classDef module fill:#a0c4ff,stroke:#333,stroke-width:1px
    classDef core fill:#d4b4ff,stroke:#333,stroke-width:1px
    classDef service fill:#ffec99,stroke:#333,stroke-width:1px
```
