# ma-da/peersai architecture

> Architecture diagram of the GitHub repository ma-da/peersai, generated by GitDiagram from its file tree, README and a sample of source files (last updated 2025-09-18). It is an AI-made overview: check details against the source.

- Interactive diagram: https://gitdiagram.com/ma-da/peersai
- Repository: https://github.com/ma-da/peersai

## Overview

##### Project type and purpose
- This is a multi-stage data pipeline plus AI training system with a client-side search frontend.
- Purpose:
- webscraper crawls and downloads raw “peers-family” corpus material.
- preprocessing ingests that raw corpus, filters and indexes it.
- IPFS processes, cleans, and uploads the indexed corpus to a decentralized storage layer.
- train_ai uses the dataset on IPFS to perform LoRA training and inference experiments.
- farsight is a browser-based single-page app that queries the IPFS-backed index for TF-IDF + BGE re-ranked search.

##### File structure
- Top-level directories map to major subsystems:
– webscraper
– preprocessing
– IPFS
– train_ai
– farsight
- Each subsystem contains scripts, configs, notebooks, and utility modules.
- There are shell scripts (crawl, convert, logs) under webscraper, Python modules for filtering/indexing under preprocessing, IPFS upload and embedding tests under IPFS, Jupyter notebooks and train scripts under train_ai, and HTML/JS under farsight.

##### README for additional insights
- The README clearly lists four projects and their roles.
raw crawl → filter/index → decentralized store → model training & frontend search.

## Diagram (Mermaid)

```mermaid
flowchart TB
    %% Data Ingestion Layer
    subgraph "Data Ingestion Layer (webscraper)"
        DI_web_scraper_base["web_scraper_base.py"]:::ingestion
        DI_config["config.py"]:::ingestion
        DI_runsh["run.sh"]:::ingestion
        DI_pdf_fetcher["pdf_fetcher.py"]:::ingestion
        DI_pdf_to_txt_conversion["pdf_to_txt_conversion.py"]:::ingestion
        DI_single_convert["single_convert.py"]:::ingestion
        DI_utils["utils.py"]:::ingestion
        DI_install_deps["install_deps.py"]:::ingestion
        DI_clean_logs["clean_logs.sh"]:::ingestion
    end

    DI_web_scraper_base -->|"produces raw_corpus.tar"| RawCorpus["raw_corpus.tar"]
    RawCorpus --> PreprocessingLayer

    %% Preprocessing & Indexing Layer
    subgraph "Preprocessing & Indexing Layer (preprocessing)"
        PP_corpus_filtering["corpus_filtering.py"]:::preprocessing
        PP_corpus_indexing["corpus_indexing.py"]:::preprocessing
        PP_do_build["do_build_filtered_corpus.py"]:::preprocessing
        PP_do_indexing["do_corpus_indexing.py"]:::preprocessing
        PP_do_upload["do_storage_upload.py"]:::preprocessing
        PP_storage["storage.py"]:::preprocessing
        PP_config["config.py"]:::preprocessing
    end

    PP_do_build -->|"filtered_corpus.json"| FilteredCorpus["filtered_corpus.json"]
    FilteredCorpus --> PP_do_indexing
    PP_do_indexing -->|"local_index.db"| LocalIndex["local_index.db"]
    LocalIndex --> PP_do_upload
    PP_do_upload -->|"CID list"| CIDList["CID list"]

    CIDList --> IPFSLayer

    %% Decentralized Storage Layer
    subgraph "Decentralized Storage Layer (IPFS)"
        IP_corpus_preprocess["corpus_preprocess.py"]:::storage
        IP_hive_requests["hive-requests.py"]:::storage
        IP_embedding_test["embedding_test.ipynb"]:::storage
        IP_AI_testing["AI Testing.ipynb"]:::storage
        IP_beta_slim["Beta Slim Dataset.ipynb"]:::storage
        IP_corpus_prep["Corpus Prep.ipynb"]:::storage
        IP_requests_dir["requests/"]:::storage
        IPFSCluster["IPFS Cluster"]:::storage
    end

    CIDList -->|"uploads"| IPFSCluster
    IPFSCluster -->|"serves URIs"| IPFSOut["HTTP Gateway"]:::storage

    IPFSOut --> TrainAILayer
    IPFSOut --> FarsightLayer

    %% Machine Learning Layer
    subgraph "Machine Learning Layer (train_ai)"
        TA_train_ai["train_ai.py"]:::training
        TA_handler["handler.py"]:::training
        TA_has_gpus["has_gpus.py"]:::training
        TA_add_stop["add_stop_token.py"]:::training
        TA_train_lora["train_lora_from_corpus*.ipynb"]:::training
        TA_infer_lora["inference_test_lora.ipynb"]:::training
        TA_infer_base["inference_test_base_model_huggingface.ipynb"]:::training
        TA_merge_lora["merge_lora_with_base_model.ipynb"]:::training
        TA_test_load["test_load_llm.py"]:::training
        TA_test_results["test_results/"]:::training
    end

    IPFSOut -->|"pulls data for training"| TA_train_ai
    TA_train_ai --> TA_test_results

    %% Client/Search Frontend
    subgraph "Client/Search Frontend (farsight SPA)"
        F_farsight_html["farsight.html"]:::frontend
        F_ipynb["farsight.ipynb"]:::frontend
        F_readme["readme.md"]:::frontend
    end

    IPFSOut -->|"fetches index and embeddings"| F_farsight_html

    %% Click Events
    click DI_web_scraper_base "https://github.com/ma-da/peersai/blob/main/webscraper/web_scraper_base.py"
    click DI_config "https://github.com/ma-da/peersai/blob/main/webscraper/config.py"
    click DI_runsh "https://github.com/ma-da/peersai/blob/main/webscraper/run.sh"
    click DI_pdf_fetcher "https://github.com/ma-da/peersai/blob/main/webscraper/pdf_fetcher.py"
    click DI_pdf_to_txt_conversion "https://github.com/ma-da/peersai/blob/main/webscraper/pdf_to_txt_conversion.py"
    click DI_single_convert "https://github.com/ma-da/peersai/blob/main/webscraper/single_convert.py"
    click DI_utils "https://github.com/ma-da/peersai/blob/main/webscraper/utils.py"
    click DI_install_deps "https://github.com/ma-da/peersai/blob/main/webscraper/install_deps.py"
    click DI_clean_logs "https://github.com/ma-da/peersai/blob/main/webscraper/clean_logs.sh"

    click PP_corpus_filtering "https://github.com/ma-da/peersai/blob/main/preprocessing/corpus_filtering.py"
    click PP_corpus_indexing "https://github.com/ma-da/peersai/blob/main/preprocessing/corpus_indexing.py"
    click PP_do_build "https://github.com/ma-da/peersai/blob/main/preprocessing/do_build_filtered_corpus.py"
    click PP_do_indexing "https://github.com/ma-da/peersai/blob/main/preprocessing/do_corpus_indexing.py"
    click PP_do_upload "https://github.com/ma-da/peersai/blob/main/preprocessing/do_storage_upload.py"
    click PP_storage "https://github.com/ma-da/peersai/blob/main/preprocessing/storage.py"
    click PP_config "https://github.com/ma-da/peersai/blob/main/preprocessing/config.py"

    click IP_corpus_preprocess "https://github.com/ma-da/peersai/blob/main/IPFS/corpus_preprocess.py"
    click IP_hive_requests "https://github.com/ma-da/peersai/blob/main/IPFS/hive-requests.py"
    click IP_embedding_test "https://github.com/ma-da/peersai/blob/main/IPFS/embedding_test.ipynb"
    click IP_AI_testing "https://github.com/ma-da/peersai/blob/main/IPFS/AI Testing.ipynb"
    click IP_beta_slim "https://github.com/ma-da/peersai/blob/main/IPFS/Beta Slim Dataset.ipynb"
    click IP_corpus_prep "https://github.com/ma-da/peersai/blob/main/IPFS/Corpus Prep.ipynb"
    click IP_requests_dir "https://github.com/ma-da/peersai/tree/main/IPFS/requests/"

    click TA_train_ai "https://github.com/ma-da/peersai/blob/main/train_ai/train_ai.py"
    click TA_handler "https://github.com/ma-da/peersai/blob/main/train_ai/handler.py"
    click TA_has_gpus "https://github.com/ma-da/peersai/blob/main/train_ai/has_gpus.py"
    click TA_add_stop "https://github.com/ma-da/peersai/blob/main/train_ai/add_stop_token.py"
    click TA_train_lora "https://github.com/ma-da/peersai/blob/main/train_ai/train_lora_from_corpus*.ipynb"
    click TA_infer_lora "https://github.com/ma-da/peersai/blob/main/train_ai/inference_test_lora.ipynb"
    click TA_infer_base "https://github.com/ma-da/peersai/blob/main/train_ai/inference_test_base_model_huggingface.ipynb"
    click TA_merge_lora "https://github.com/ma-da/peersai/blob/main/train_ai/merge_lora_with_base_model.ipynb"
    click TA_test_load "https://github.com/ma-da/peersai/blob/main/train_ai/test_load_llm.py"
    click TA_test_results "https://github.com/ma-da/peersai/tree/main/train_ai/test_results/"

    click F_farsight_html "https://github.com/ma-da/peersai/blob/main/farsight/farsight.html"
    click F_ipynb "https://github.com/ma-da/peersai/blob/main/farsight/farsight.ipynb"
    click F_readme "https://github.com/ma-da/peersai/blob/main/farsight/readme.md"

    %% Styles
    classDef ingestion fill:#cce5ff,stroke:#333,stroke-width:1px
    classDef preprocessing fill:#d4edda,stroke:#333,stroke-width:1px
    classDef storage fill:#ffe5b4,stroke:#333,stroke-width:1px
    classDef training fill:#e2d6f3,stroke:#333,stroke-width:1px
    classDef frontend fill:#d1ecf1,stroke:#333,stroke-width:1px
```
