commit 63855f2b3aa4ac9b7758d7432e0026da001978a9 Author: GAJ3T Date: Mon Oct 26 03:35:42 2026 -0400 Copying Repo to Demo diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..27d2442 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.env +.git +.github +.venv +__pycache__ +*.py[cod] +.pytest_cache +.coverage +artifacts +build +dist +*.egg-info +tests diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5231978 --- /dev/null +++ b/.env.example @@ -0,0 +1,47 @@ +# Storage. Replace the host, user, password, and database for each environment. +POSTGRES_PASSWORD=knowledgegraph-dev +POSTGRES_PORT=5432 +API_PORT=8080 +DATABASE_URL=postgresql+psycopg://knowledgegraph_app:knowledgegraph-dev@localhost:5432/knowledgegraph +SOURCE_DATABASE_URL= +SOURCE_TABLE_NAME=external_documents +ANALYTICS_DATABASE_URL= +AUTO_CREATE_SCHEMA=false + +# External handoff and corpus. +CORPUS_NAME=product-knowledge + +# Runtime model templates. JSON files passed with --model-file override these values. +ENTITY_TYPES=Person,Organization,Product,Concept,Event,Location +RELATIONSHIP_TYPES=AFFILIATED_WITH,CREATED,DEPENDS_ON,MENTIONS,PART_OF,RELATED_TO +NODE_TEMPLATE={"canonical_name":"string","entity_type":"one configured entity type","description":"source-grounded string","aliases":["string"]} +EDGE_TEMPLATE={"source":"entity name","target":"entity name","relationship_type":"one configured relationship type","description":"source-grounded string"} +EXTRACTION_INSTRUCTIONS=Extract only entities and relationships explicitly supported by the supplied text. + +# OpenAI-compatible chat and embedding provider. +LLM_BASE_URL=https://api.openai.com/v1 +LLM_API_KEY=replace-with-provider-key +LLM_CHAT_MODEL=gpt-4.1-mini +LLM_EMBEDDING_MODEL=text-embedding-3-small +LLM_EMBEDDING_DIMENSIONS=1536 +LLM_TIMEOUT_SECONDS=60 +LLM_MAX_RETRIES=4 +LLM_RETRY_INITIAL_SECONDS=1 +LLM_RETRY_MAX_SECONDS=30 + +# Processing and retrieval. +CHUNK_SIZE=1200 +CHUNK_OVERLAP=150 +COMMUNITY_RESOLUTION=1.0 +RETRIEVAL_TOP_K=8 +RETRIEVAL_GRAPH_HOPS=2 +RETRIEVAL_GRAPH_LIMIT=50 +RETRIEVAL_HNSW_EF_SEARCH=100 +LOG_LEVEL=INFO +INTERNAL_JWT_SECRET=replace-with-a-dev-test-secret-at-least-32-bytes +INTERNAL_JWT_ISSUER=knowledgegraph-bff +INTERNAL_JWT_AUDIENCE=knowledgegraph-service +INTERNAL_FEATURE_NAME=KnowledgeGraph +WORKER_POLL_SECONDS=2 +WORKER_STALE_RUN_SECONDS=3600 +ARTIFACT_DIR=artifacts \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3207bca --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,56 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_USER: knowledgegraph_app + POSTGRES_PASSWORD: test-password + POSTGRES_DB: knowledgegraph + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U knowledgegraph_app -d knowledgegraph" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + + env: + DATABASE_URL: postgresql+psycopg://knowledgegraph_app:test-password@localhost:5432/knowledgegraph + TEST_POSTGRES_URL: postgresql+psycopg://knowledgegraph_app:test-password@localhost:5432/knowledgegraph + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install + run: python -m pip install -e ".[test]" + + - name: Compile + run: python -m compileall -q src tests migrations + + - name: Check dependencies + run: python -m pip check + + - name: Run portable tests + run: python -m pytest -q --ignore=tests/integration/test_pgvector_postgres.py + + - name: Run migration and PgVector integration + run: python -m pytest tests/integration/test_pgvector_postgres.py -q + + - name: Validate Compose + run: docker compose config --quiet + + - name: Build application image + run: docker build -f Containerfile -t knowledgegraph:test . diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..255642b --- /dev/null +++ b/.gitignore @@ -0,0 +1,12 @@ +.venv/ +__pycache__/ +*.py[cod] +.pytest_cache/ +.coverage +htmlcov/ +.env +*.db +artifacts/ +dist/ +build/ +*.egg-info/ \ No newline at end of file diff --git a/Containerfile b/Containerfile new file mode 100644 index 0000000..3de9289 --- /dev/null +++ b/Containerfile @@ -0,0 +1,26 @@ +FROM python:3.12-slim AS runtime + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +RUN groupadd --system knowledgegraph \ + && useradd --system --gid knowledgegraph --create-home knowledgegraph + +WORKDIR /app + +COPY pyproject.toml README.md alembic.ini ./ +COPY migrations ./migrations +COPY src ./src + +RUN python -m pip install --upgrade pip \ + && python -m pip install . + +RUN mkdir -p /app/artifacts \ + && chown -R knowledgegraph:knowledgegraph /app + +USER knowledgegraph + +EXPOSE 8080 + +CMD ["uvicorn", "graphrag_platform.api:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/README.md b/README.md new file mode 100644 index 0000000..539820b --- /dev/null +++ b/README.md @@ -0,0 +1,645 @@ +# KnowledgeGraph GraphRAG + +An internal, restartable GraphRAG pipeline that normalizes externally collected +content, extracts and reconciles knowledge, builds a graph, indexes graph and source +evidence with PostgreSQL/PgVector, retrieves graph-aware context, and generates +source-cited answers. + +The collection process is outside this project. Existing **Web-Access MCP** and +**Data-Store MCP** processes discover and persist source material. This project +starts by validating and normalizing their versioned handoff. + +## Architecture + +```text +Web-Access MCP + Data-Store MCP + | + | versioned JSONL or PostgreSQL handoff + v +01 normalize and parse + v +02 clean, deduplicate, and chunk + v +03 define knowledge model + v +04 extract entities and relationships + v +05 reconcile aliases and duplicates + v +06 materialize graph nodes and edges + v +07 detect and summarize communities + v +08 build PgVector HNSW retrieval index + v +09 retrieve vector + graph + source context + v +10 generate a cited answer +``` + +PostgreSQL is the canonical checkpoint store. Every completed build stage persists +its output, so later stages can be rebuilt without repeating valid upstream work. +PgVector stores embeddings and performs server-side HNSW cosine search. Indexed +relational edge tables provide bounded graph expansion. + +The deployment consists of: + +- PostgreSQL 16 with PgVector +- Alembic migration job +- Internal FastAPI service +- PostgreSQL-backed asynchronous worker +- Existing BFF as the policy enforcement point + +The API is internal only and is not intended for direct customer access. + +## Repository layout + +| Path | Purpose | +|---|---| +| [`src/01_normalize_content.py`](src/01_normalize_content.py) through [`src/10_generate_answer.py`](src/10_generate_answer.py) | Complete numbered stage implementations | +| [`src/graphrag_platform`](src/graphrag_platform) | Configuration, models, storage, orchestration, API, auth, worker, and provider clients | +| [`migrations`](migrations) | Alembic database migrations | +| [`tests`](tests) | Unit, API/worker, pipeline, migration, and real PgVector integration tests | +| [`compose.yaml`](compose.yaml) | Reproducible PostgreSQL, migration, API, and worker environment | +| [`Containerfile`](Containerfile) | Non-root application image | +| [`.env.example`](.env.example) | Runtime configuration template | + +Detailed architecture, API, runbook, configuration, data model, testing, and +editable C4 diagrams are available in [`docs/`](docs/README.md). + +## Prerequisites + +- Python 3.11 or newer +- Podman 5 or newer +- `podman machine` running on Windows +- An OpenAI-compatible chat and embeddings endpoint +- External Web-Access MCP/Data-Store MCP collection process + +## Quick start with Compose + +Create the local environment: + +```powershell +Copy-Item .env.example .env +``` + +Edit `.env` and set at minimum: + +```dotenv +POSTGRES_PASSWORD=choose-a-local-password +DATABASE_URL=postgresql+psycopg://knowledgegraph_app:choose-a-local-password@localhost:5432/knowledgegraph + +LLM_BASE_URL=https://api.openai.com/v1 +LLM_API_KEY=replace-with-provider-key +LLM_CHAT_MODEL=gpt-4.1-mini +LLM_EMBEDDING_MODEL=text-embedding-3-small +LLM_EMBEDDING_DIMENSIONS=1536 + +INTERNAL_JWT_SECRET=replace-with-at-least-32-random-bytes +``` + +`POSTGRES_PASSWORD` and the password in `DATABASE_URL` must match. Compose replaces +the database host with `postgres` inside application containers. + +Start the complete environment: + +```powershell +podman machine start +podman compose up --detach --build +``` + +Compose: + +1. Starts `docker.io/pgvector/pgvector:pg16`. +2. Waits for PostgreSQL readiness. +3. Runs `alembic upgrade head`. +4. Starts the API and worker. +5. Waits for API dependency readiness. + +Verify it: + +```powershell +podman compose ps +Invoke-RestMethod http://127.0.0.1:8080/internal/knowledge-graph/health +Invoke-RestMethod http://127.0.0.1:8080/internal/knowledge-graph/ready +``` + +View logs: + +```powershell +podman compose logs --follow api worker +``` + +Stop services while retaining PostgreSQL data: + +```powershell +podman compose down +``` + +Delete services and all local database data: + +```powershell +podman compose down --volumes +``` + +Volume removal is destructive. + +## External source handoff + +### JSON Lines contract + +Each UTF-8 JSONL line must satisfy schema version `1.0`: + +```json +{ + "schema_version": "1.0", + "external_id": "stable-source-record-id", + "corpus": "product-knowledge", + "url": "https://docs.example.com/article", + "body": "
Article text
", + "content_type": "text/html", + "status_code": 200, + "retrieved_at": "2026-10-13T18:00:00Z", + "metadata": { + "collector": "Web-Access", + "stored_by": "Data-Store" + } +} +``` + +Required fields: + +- `schema_version`: currently `1.0` +- `external_id`: stable and unique within a corpus +- `url` +- `body` +- `retrieved_at`: ISO 8601 timestamp with offset + +Optional/defaulted fields: + +- `corpus`: when provided, it must match the selected runtime corpus +- `content_type`: defaults to `text/html` +- `status_code`: defaults to `200` +- `metadata`: defaults to an empty object + +Unknown fields and unsupported schema versions are rejected. Invalid records include +the file and line number in the error. + +The template is [`src/templates/mcp_documents.jsonl`](src/templates/mcp_documents.jsonl). +A larger acceptance corpus is +[`tests/fixtures/external_documents.jsonl`](tests/fixtures/external_documents.jsonl). + +### PostgreSQL source contract + +Set `SOURCE_DATABASE_URL` and `SOURCE_TABLE_NAME` to read the same contract from an +external PostgreSQL table: + +```dotenv +SOURCE_DATABASE_URL=postgresql+psycopg://source_reader:password@source-host:5432/source_database +SOURCE_TABLE_NAME=external_documents +``` + +Compatible table: + +```sql +CREATE TABLE external_documents ( + external_id text PRIMARY KEY, + schema_version text NOT NULL DEFAULT '1.0', + corpus text, + url text NOT NULL, + body text NOT NULL, + content_type text NOT NULL DEFAULT 'text/html', + status_code integer NOT NULL DEFAULT 200, + retrieved_at timestamptz NOT NULL, + metadata jsonb NOT NULL DEFAULT '{}'::jsonb +); +``` + +Additional source-table columns are ignored. The application account should receive +read-only access to the source table. + +Provide either `--input-jsonl` or `SOURCE_DATABASE_URL`, not both. + +## Durable checkpoints + +Build stages persist these boundaries: + +```text +raw_documents + documents + -> chunks + -> knowledge_models + -> extractions + -> resolved_entities + resolved_relationships + -> nodes + edges + -> communities + -> retrieval_items +``` + +`pipeline_runs` and `stage_runs` separately record: + +- queued/running/succeeded/failed state +- selected stage range +- attempt +- configuration fingerprint +- caller and correlation ID +- stage inputs and outputs +- timestamps and error details + +The worker requeues runs abandoned beyond `WORKER_STALE_RUN_SECONDS`. Set this value +above the expected maximum duration of one stage. + +PostgreSQL advisory locks allow only one active build for a corpus. If another +worker claims a job for that corpus, the job is returned to the queue instead of +executing conflicting writes. + +DataFrames are not stage checkpoints. A Pandas or Polars DataFrame may be used by a +future adapter for one bounded transformation, but every completed result must be +persisted before the next stage starts. + +## Database migrations + +Runtime services verify the migrated schema and do not silently mutate it. + +Apply migrations: + +```powershell +alembic upgrade head +``` + +Inspect the current revision: + +```powershell +alembic current +alembic history +``` + +Create a migration after changing SQLAlchemy models: + +```powershell +alembic revision --autogenerate -m "describe schema change" +``` + +Review generated migrations before applying them. Compose runs upgrades through the +one-shot `migrate` service before starting API and worker services. + +`AUTO_CREATE_SCHEMA=true` exists only for isolated tests. Do not enable it in shared +dev, test, or production environments. + +## Run stages directly + +Install a local editable environment: + +```powershell +py -3.12 -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +python -m pip install -e ".[test]" +``` + +Run migrations before the first command: + +```powershell +alembic upgrade head +``` + +### 1. Normalize external content + +```powershell +python .\src\01_normalize_content.py ` + --corpus product-knowledge ` + --input-jsonl .\artifacts\mcp-documents.jsonl +``` + +This validates the external handoff, stores raw provenance, runs Trafilatura for +HTML, and persists normalized document records. + +### 2. Clean, deduplicate, and chunk + +```powershell +python .\src\02_clean_content.py --corpus product-knowledge +``` + +### 3. Define the knowledge model + +```powershell +python .\src\03_define_knowledge_model.py ` + --corpus product-knowledge ` + --model-file .\src\templates\knowledge_model.json +``` + +### 4. Extract entities and relationships + +```powershell +python .\src\04_extract_knowledge.py --corpus product-knowledge +``` + +LLM calls occur outside long database transactions. Each validated chunk result is +committed independently. + +### 5. Reconcile knowledge + +```powershell +python .\src\05_reconcile_knowledge.py --corpus product-knowledge +``` + +### 6. Build graph nodes and edges + +```powershell +python .\src\06_build_knowledge_graph.py --corpus product-knowledge +``` + +### 7. Build communities + +```powershell +python .\src\07_build_communities.py --corpus product-knowledge +``` + +### 8. Build the PgVector index + +```powershell +python .\src\08_build_retrieval_index.py --corpus product-knowledge +``` + +This embeds chunks, nodes, edges, and community summaries, validates dimensions, +stores vectors, and creates a dimension-specific HNSW cosine index. + +### 9. Retrieve GraphRAG context + +```powershell +python .\src\09_query_graphrag.py ` + --corpus product-knowledge ` + --question "Who created Atlas?" ` + --top-k 8 +``` + +The result contains vector `context`, expanded `graph_context`, and traceable +`sources`. + +### 10. Generate a cited answer + +```powershell +python .\src\10_generate_answer.py ` + --corpus product-knowledge ` + --question "Who created Atlas?" +``` + +## Run a durable build + +Run stages 1 through 8 under one durable pipeline record: + +```powershell +knowledgegraph run ` + --corpus product-knowledge ` + --from-stage 1 ` + --through-stage 8 ` + --input-jsonl .\artifacts\mcp-documents.jsonl ` + --model-file .\src\templates\knowledge_model.json ` + --requested-by local-developer +``` + +Use `--from-stage` to rebuild only downstream state. The `--reset` option deletes +the selected stage and downstream derived corpus data before execution. + +## BFF authorization contract + +The existing BFF authenticates callers and evaluates the `KnowledgeGraph` feature. +Independent permission bits are: + +```text +Read = 1 +Add = 2 +Edit = 4 +Delete = 8 +Execute = 16 +``` + +| Operation | Required mask | +|---|---:| +| Read job/source/retrieval information | `Read` (`1`) | +| Submit a stage-1 source job | `Add | Execute` (`18`) | +| Run downstream build stages | `Execute` (`16`) | +| Query or answer | `Read | Execute` (`17`) | +| Retry a failed job | `Execute` (`16`) | + +The BFF forwards a signed internal JWT. Required claims: + +```json +{ + "iss": "knowledgegraph-bff", + "aud": "knowledgegraph-service", + "sub": "internal-caller-id", + "iat": 1791930000, + "exp": 1791930600, + "jti": "correlation-or-request-id", + "features": { + "KnowledgeGraph": 31 + } +} +``` + +Dev/test uses HS256 and `INTERNAL_JWT_SECRET`. Production deployments can replace +this verifier with the BFF's workload identity, mTLS, or asymmetric/JWKS contract. +Never trust unsigned permission headers. + +## Internal API + +| Endpoint | Purpose | +|---|---| +| `GET /internal/knowledge-graph/health` | Process liveness | +| `GET /internal/knowledge-graph/ready` | Database and PgVector readiness | +| `POST /internal/knowledge-graph/jobs` | Queue stages 1–8 | +| `GET /internal/knowledge-graph/jobs/{jobId}` | Read durable job/stage status | +| `POST /internal/knowledge-graph/jobs/{jobId}/retry` | Requeue a failed job | +| `POST /internal/knowledge-graph/query` | Synchronous bounded retrieval | +| `POST /internal/knowledge-graph/answer` | Synchronous cited answer | + +Example job request: + +```json +{ + "corpus": "product-knowledge", + "from_stage": 1, + "through_stage": 8, + "input_jsonl": "/app/artifacts/mcp-documents.jsonl", + "model_file": "/app/src/templates/knowledge_model.json", + "reset": false +} +``` + +Compose bind-mounts local [`artifacts`](artifacts) at `/app/artifacts`, allowing the +worker to read JSONL handoffs placed there. In deployed environments, prefer the +PostgreSQL source contract or a shared managed file volume. + +Long-running stages return `202 Accepted` with a job ID. Query and answer requests +remain synchronous and bounded. + +## Retry and logging behavior + +Model-provider calls retry: + +- HTTP 429 +- HTTP 500, 502, 503, and 504 +- connection failures +- timeouts + +Retries honor numeric `Retry-After`, otherwise use bounded exponential backoff with +jitter. Authentication and validation errors are not retried. + +Logs are emitted as one JSON object per line and include event names, run IDs, +corpus/stage context where available, attempts, status, and error details. Secrets +are not logged. + +## Tests + +Portable tests: + +```powershell +pytest --ignore=.\tests\integration\test_pgvector_postgres.py +``` + +Real PostgreSQL/PgVector migration and retrieval test: + +```powershell +$env:TEST_POSTGRES_URL = "postgresql+psycopg://knowledgegraph_app:password@localhost:5432/knowledgegraph" +pytest .\tests\integration\test_pgvector_postgres.py +Remove-Item Env:TEST_POSTGRES_URL +``` + +The suite covers: + +- Versioned source validation +- Nested metadata and quoted/comma-containing content +- Configuration parsing +- Chunk validation +- Cross-chunk relationship reconciliation +- Provider retry policy +- Full ten-stage behavior +- Signed BFF authorization and permission failures +- Durable API job and worker execution +- Alembic migration against PostgreSQL +- PgVector extension and HNSW index creation +- Server-side vector retrieval, graph expansion, and citations + +Opt-in vector load test: + +```powershell +$env:TEST_POSTGRES_URL = "postgresql+psycopg://knowledgegraph_app:password@localhost:5432/knowledgegraph" +$env:RUN_PERFORMANCE_TESTS = "1" +$env:PERF_VECTOR_ITEM_COUNT = "10000" +$env:PERF_MAX_QUERY_SECONDS = "2.0" +pytest .\tests\performance\test_vector_query_load.py +``` + +Set the item count and latency threshold to the capacity target for the dev/test +environment. Performance checks are excluded by default because machine resources +vary. + +CI is defined in [`.github/workflows/ci.yml`](.github/workflows/ci.yml) and compiles, +tests, validates Compose, exercises real PgVector, and builds the application image. + +## Persistent format policy + +- PostgreSQL is the canonical operational store. +- JSON Lines is the preferred file handoff/checkpoint format. +- CSV is allowed only for explicitly defined flat import/export views. + +CSV safely supports commas, quotes, and line breaks when written by a compliant CSV +library. It remains unsuitable as the canonical graph format because aliases, +metadata, source ID arrays, embeddings, and graph properties are nested and typed. +Never construct CSV by joining strings with commas. + +## RAG database alternatives + +### PostgreSQL with PgVector + +Implemented default. Choose it for SQL transactions, colocated provenance and +vectors, self-managed deployment, and tunable HNSW retrieval. Graph traversal uses +indexed relational edges rather than a native property-graph language. + +Official reference: [PgVector](https://github.com/pgvector/pgvector). + +### Azure Cosmos DB for NoSQL + +Choose it for managed/serverless document storage, vector indexing, and first-party +vector plus BM25 hybrid search. Cosmos DB's Gremlin API is separate from the NoSQL +vector feature; use document adjacency or a separate graph store. + +References: + +- [Cosmos DB vector search](https://learn.microsoft.com/azure/cosmos-db/vector-search) +- [Cosmos DB hybrid search](https://learn.microsoft.com/azure/cosmos-db/gen-ai/hybrid-search) +- [Cosmos DB Gremlin API](https://learn.microsoft.com/azure/cosmos-db/gremlin/overview) + +### Neo4j + +Choose it when graph topology, paths, and multi-hop traversal are primary retrieval +signals. Neo4j combines native property graphs, Cypher, vector indexes, and +graph-aware hybrid retrieval. It requires a dedicated backend adapter; changing +`DATABASE_URL` is not sufficient. Confirm version and edition requirements for the +selected vector and unified-search features. + +References: + +- [Neo4j vector indexes](https://neo4j.com/docs/cypher-manual/current/indexes/search-indexes/vector-indexes/) +- [Neo4j hybrid search](https://neo4j.com/developer/genai-ecosystem/hybrid-search/) + +Other credible options: + +- [Amazon Neptune Analytics](https://docs.aws.amazon.com/neptune-analytics/latest/userguide/what-is-neptune-analytics.html) + for AWS-managed graph/vector analytics +- [Azure AI Search](https://learn.microsoft.com/azure/search/hybrid-search-overview) + as a secondary managed hybrid index without a native graph model + +## Operational guidance + +- Keep the API and worker on internal networks accessible only to the BFF and + operators. +- Replace all dev secrets before shared deployment. +- Use a separate PostgreSQL role for migrations and a least-privilege runtime role. +- Back up PostgreSQL before destructive resets or migrations. +- Monitor queued/running/failed run counts, stage duration, LLM retries, database + pool usage, vector latency, and retrieval result counts. +- Set `WORKER_STALE_RUN_SECONDS` above the longest expected single stage. +- Rebuild stage 8 after changing embedding model or dimensions. +- Maintain a golden question/citation set to detect retrieval-quality regressions. + +## Troubleshooting + +### Database migrations are required + +```powershell +alembic current +alembic upgrade head +``` + +### `LLM_API_KEY is required` + +Set `LLM_API_KEY` in `.env` and restart API/worker containers. + +### Embedding dimension mismatch + +Set `LLM_EMBEDDING_DIMENSIONS` to the exact provider output dimension, then reset +and rerun stage 8. + +### Source handoff rejected + +Validate every JSONL line against schema version `1.0`. Confirm `corpus`, +`external_id`, URL, body, and `retrieved_at`. + +### Job remains queued + +```powershell +podman compose ps worker +podman compose logs --tail 100 worker +``` + +### PostgreSQL/PgVector is not ready + +```powershell +podman compose ps postgres +podman compose logs --tail 100 postgres migrate +``` + +### Port conflict + +Change `POSTGRES_PORT` or `API_PORT` in `.env`, then recreate the Compose stack. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..1f3d3e9 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,38 @@ +[alembic] +script_location = migrations +prepend_sys_path = . +path_separator = os + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..5ffe829 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,99 @@ +name: knowledgegraph + +services: + postgres: + image: docker.io/pgvector/pgvector:pg16 + environment: + POSTGRES_USER: knowledgegraph_app + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-knowledgegraph-dev} + POSTGRES_DB: knowledgegraph + ports: + - "127.0.0.1:${POSTGRES_PORT:-5432}:5432" + volumes: + - knowledgegraph-postgres-data:/var/lib/postgresql/data + healthcheck: + test: + [ + "CMD-SHELL", + "pg_isready -U knowledgegraph_app -d knowledgegraph", + ] + interval: 5s + timeout: 5s + retries: 20 + restart: unless-stopped + + migrate: + build: + context: . + dockerfile: Containerfile + command: ["alembic", "upgrade", "head"] + environment: &application-environment + DATABASE_URL: postgresql+psycopg://knowledgegraph_app:${POSTGRES_PASSWORD:-knowledgegraph-dev}@postgres:5432/knowledgegraph + SOURCE_DATABASE_URL: ${SOURCE_DATABASE_URL:-} + SOURCE_TABLE_NAME: ${SOURCE_TABLE_NAME:-external_documents} + AUTO_CREATE_SCHEMA: "false" + CORPUS_NAME: ${CORPUS_NAME:-default-corpus} + LLM_BASE_URL: ${LLM_BASE_URL:-https://api.openai.com/v1} + LLM_API_KEY: ${LLM_API_KEY:-} + LLM_CHAT_MODEL: ${LLM_CHAT_MODEL:-gpt-4.1-mini} + LLM_EMBEDDING_MODEL: ${LLM_EMBEDDING_MODEL:-text-embedding-3-small} + LLM_EMBEDDING_DIMENSIONS: ${LLM_EMBEDDING_DIMENSIONS:-1536} + INTERNAL_JWT_SECRET: ${INTERNAL_JWT_SECRET:-knowledgegraph-dev-secret-change-this-32} + INTERNAL_JWT_ISSUER: ${INTERNAL_JWT_ISSUER:-knowledgegraph-bff} + INTERNAL_JWT_AUDIENCE: ${INTERNAL_JWT_AUDIENCE:-knowledgegraph-service} + INTERNAL_FEATURE_NAME: ${INTERNAL_FEATURE_NAME:-KnowledgeGraph} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api: + build: + context: . + dockerfile: Containerfile + command: + [ + "uvicorn", + "graphrag_platform.api:app", + "--host", + "0.0.0.0", + "--port", + "8080", + ] + environment: *application-environment + ports: + - "127.0.0.1:${API_PORT:-8080}:8080" + volumes: + - ./artifacts:/app/artifacts + depends_on: + migrate: + condition: service_completed_successfully + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/internal/knowledge-graph/ready', timeout=5)", + ] + interval: 10s + timeout: 5s + retries: 12 + restart: unless-stopped + + worker: + build: + context: . + dockerfile: Containerfile + command: ["knowledgegraph-worker"] + environment: *application-environment + volumes: + - ./artifacts:/app/artifacts + depends_on: + migrate: + condition: service_completed_successfully + restart: unless-stopped + +volumes: + knowledgegraph-postgres-data: diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 0000000..f6e5055 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,54 @@ +# KnowledgeGraph GraphRAG documentation + +This folder is the operational and technical reference for the KnowledgeGraph +GraphRAG proof of concept. The repository README remains the shortest getting +started path; these documents explain the contracts behind it. + +## Read this first + +| Document | Use it for | +|---|---| +| [System overview](./overview.md) | Scope, boundaries, stage flow, and key design decisions | +| [Architecture](./architecture.md) | Runtime topology, stage contracts, persistence, and retrieval | +| [Runbook](./runbook.md) | Step-by-step local setup, ingestion, rebuilds, recovery, and shutdown | +| [Configuration](./configuration.md) | Environment variables, defaults, and production guidance | +| [Source contract](./source-contract.md) | JSONL and PostgreSQL handoff schemas | +| [API reference](./api.md) | Internal endpoints, JWT claims, permissions, and examples | +| [Data model](./data-model.md) | Checkpoint tables and stage ownership | +| [Testing](./testing.md) | Test commands, prerequisites, and performance checks | +| [Operations and troubleshooting](./operations.md) | Logs, health checks, failures, retries, and known limitations | +| [Module reference](./module-reference.md) | Python modules, stage entry points, and responsibilities | + +## C4 diagrams + +The diagrams are editable in [diagrams.net/draw.io](https://app.diagrams.net/). +Each has a Mermaid companion with matching nodes, relationships, and colors, +plus a PDF exported directly from the draw.io source. Mermaid uses automatic +layout, so its positioning may differ from the draw.io diagram. + +| C4 level | Draw.io source | Mermaid source | PDF | +|---|---|---|---| +| Context | [Draw.io](./diagrams/c4-context.drawio) | [Mermaid](./diagrams/c4-context.mmd) | [PDF](./diagrams/c4-context.pdf) | +| Container | [Draw.io](./diagrams/c4-container.drawio) | [Mermaid](./diagrams/c4-container.mmd) | [PDF](./diagrams/c4-container.pdf) | +| Component | [Draw.io](./diagrams/c4-component.drawio) | [Mermaid](./diagrams/c4-component.mmd) | [PDF](./diagrams/c4-component.pdf) | +| Code | [Draw.io](./diagrams/c4-code.drawio) | [Mermaid](./diagrams/c4-code.mmd) | [PDF](./diagrams/c4-code.pdf) | + +After editing a draw.io source, update its Mermaid companion and regenerate its +PDF with the draw.io desktop CLI. For example, from the repository root: + +```powershell +drawio --export --format pdf --crop --output .\docs\diagrams\c4-context.pdf .\docs\diagrams\c4-context.drawio +``` + +## Documentation scope + +The documentation is based on the implementation, tests, migration, Compose +definition, CI workflow, templates, and repository README. It documents current +behavior rather than promising features that are not implemented. In particular: + +- source collection is external to this repository; +- stages 1-8 are durable worker jobs; +- query and answer are synchronous API/stage operations; +- PostgreSQL with PgVector is the deployment target; +- SQLite is supported for isolated tests, not production retrieval; +- LLM output is schema-validated but not independently fact-checked. diff --git a/docs/api.md b/docs/api.md new file mode 100644 index 0000000..b143544 --- /dev/null +++ b/docs/api.md @@ -0,0 +1,71 @@ +# Internal API reference + +All routes are under `/internal/knowledge-graph`. Health is unauthenticated; +other routes require an internal signed JWT from the BFF. + +## Endpoints + +| Method and path | Permission | Behavior | +|---|---:|---| +| `GET /health` | none | process liveness | +| `GET /ready` | none | database and PgVector readiness | +| `POST /jobs` | stage 1: `18`; downstream: `16` | queue stages 1-8 | +| `GET /jobs/{jobId}` | `1` | durable job and stage status | +| `POST /jobs/{jobId}/retry` | `16` | requeue a failed job | +| `POST /query` | `17` | synchronous retrieval | +| `POST /answer` | `17` | synchronous retrieval plus cited answer | + +Permission bits are `Read=1`, `Add=2`, `Edit=4`, `Delete=8`, and `Execute=16`. + +## Job request + +```json +{ + "corpus": "product-knowledge", + "from_stage": 1, + "through_stage": 8, + "input_jsonl": "/app/artifacts/mcp-documents.jsonl", + "model_file": "/app/src/templates/knowledge_model.json", + "reset": false +} +``` + +Long-running job submission returns `202 Accepted` with a `job_id`. Query and +answer remain synchronous and bounded. + +## Query and answer requests + +```json +{ + "corpus": "product-knowledge", + "question": "Who created Atlas?", + "top_k": 8 +} +``` + +Retrieval returns vector `context`, bounded `graph_context`, and traceable +`sources`. Answer generation asks the LLM to cite those sources as `[S1]`, +`[S2]`, and so on. + +## JWT contract + +The verifier expects HS256 in the current implementation and validates `iss`, +`aud`, `sub`, `iat`, `exp`, `jti`, and a feature permission mask: + +```json +{ + "iss": "knowledgegraph-bff", + "aud": "knowledgegraph-service", + "sub": "internal-caller-id", + "iat": 1791930000, + "exp": 1791930600, + "jti": "request-id", + "features": { + "KnowledgeGraph": 31 + } +} +``` + +The BFF remains the policy enforcement point. Do not substitute unsigned +headers for the token. + diff --git a/docs/architecture.md b/docs/architecture.md new file mode 100644 index 0000000..e220c6c --- /dev/null +++ b/docs/architecture.md @@ -0,0 +1,88 @@ +# Architecture + +## Logical architecture + +```text +Web-Access/Data-Store MCP + | + | JSONL file or PostgreSQL source table + v +StageContext + numbered stages + | + +--> PostgreSQL checkpoint tables + +--> PgVector retrieval_items + HNSW index + +--> structured JSON logs and artifacts + | + +--> OpenAI-compatible chat/embedding provider + +BFF -- signed JWT --> FastAPI API --> orchestrator --> worker --> stages + | + +--> query/answer stages +``` + +See the editable [C4 context diagram](./diagrams/c4-context.drawio) and +[C4 container diagram](./diagrams/c4-container.drawio) for the system boundary. + +## Stage contracts + +| Stage | Reads | Writes | Result | +|---:|---|---|---| +| 1 | JSONL or source table | `raw_documents`, `documents` | normalized documents | +| 2 | documents | `chunks` | cleaned, deduplicated chunks | +| 3 | settings/model file | `knowledge_models` | entity and relationship vocabulary | +| 4 | chunks, model | `extractions` | validated LLM extraction payloads | +| 5 | extractions | `resolved_entities`, `resolved_relationships` | canonical entities and links | +| 6 | resolved data | `nodes`, `edges` | graph materialization | +| 7 | graph | `communities` | detected communities and LLM summaries | +| 8 | chunks, graph, communities | `search_items` | embeddings and HNSW index | +| 9 | search index, graph, documents | none | vector context, graph context, sources | +| 10 | stage 9 result | none | cited answer text and sources | + +Stage 8 indexes chunks, nodes, edges, and community summaries. Stage 9 embeds +the question, retrieves `top_k` items, expands graph neighbors for the +configured number of hops and limit, and resolves source URLs. + +## Durable execution + +`pipeline_runs` and `stage_runs` are separate from stage data. A run can be: + +```text +queued -> running -> succeeded + \-> failed -> queued (retry) +``` + +The worker: + +1. initializes and verifies the schema; +2. requeues stale runs older than `WORKER_STALE_RUN_SECONDS`; +3. claims one queued run using database locking; +4. takes a per-corpus PostgreSQL advisory lock; +5. executes the selected stage range; +6. persists stage status and outputs; +7. releases the lock. + +If a corpus lock is unavailable, execution is deferred and the job remains +queued. Retry resumes after the last successful stage unless the caller uses +`--reset` or otherwise clears downstream state. + +## Deployment topology + +`compose.yaml` starts PostgreSQL, waits for its health check, runs the one-shot +`migrate` service, then starts `api` and `worker`. The API and worker share the +application environment and bind-mount `./artifacts` into `/app/artifacts`. + +The [C4 container diagram](./diagrams/c4-container.drawio) shows these +containers and external dependencies. The [component diagram](./diagrams/c4-component.drawio) +maps the application package, and the [code diagram](./diagrams/c4-code.drawio) +maps the numbered stage pipeline. + +## Failure and consistency model + +- Source records are rejected before they become normalized documents if their + schema, required fields, or corpus are invalid. +- A completed stage is a checkpoint, not a transaction spanning the whole build. +- A failed stage may leave partial stage side effects; reset the affected stage + and downstream state when a clean rebuild is required. +- PostgreSQL is required to exercise PgVector and HNSW behavior. +- Embedding vector length must equal `LLM_EMBEDDING_DIMENSIONS`. + diff --git a/docs/configuration.md b/docs/configuration.md new file mode 100644 index 0000000..4d3a9a2 --- /dev/null +++ b/docs/configuration.md @@ -0,0 +1,79 @@ +# Configuration reference + +`Settings` reads environment variables and `.env` values. The complete starter +set is in [`.env.example`](../.env.example). Values below are grouped by +responsibility; names are case-insensitive through Pydantic settings. + +## Database and source + +| Variable | Purpose | Default | +|---|---|---| +| `DATABASE_URL` | canonical checkpoint database | PostgreSQL on localhost | +| `SOURCE_DATABASE_URL` | optional external source database | empty | +| `SOURCE_TABLE_NAME` | source table | `external_documents` | +| `ANALYTICS_DATABASE_URL` | reserved analytics setting | empty | +| `AUTO_CREATE_SCHEMA` | test-only schema creation | `false` | +| `ARTIFACT_DIR` | local handoff/artifact directory | `artifacts` | + +## Corpus and model + +| Variable | Purpose | Default | +|---|---|---| +| `CORPUS_NAME` | corpus namespace | `default-corpus` | +| `ENTITY_TYPES` | comma-separated allowed entity types | Person, Organization, Product, Concept, Event, Location | +| `RELATIONSHIP_TYPES` | comma-separated allowed edge types | AFFILIATED_WITH, CREATED, DEPENDS_ON, MENTIONS, PART_OF, RELATED_TO | +| `NODE_TEMPLATE` | JSON extraction shape | template in `.env.example` | +| `EDGE_TEMPLATE` | JSON extraction shape | template in `.env.example` | +| `EXTRACTION_INSTRUCTIONS` | additional extraction guardrail | source-grounded extraction | + +Use [the model template](../src/templates/knowledge_model.json) when invoking +stage 3 with `--model-file`. + +## LLM provider + +| Variable | Purpose | Default | +|---|---|---| +| `LLM_BASE_URL` | OpenAI-compatible API root | `https://api.openai.com/v1` | +| `LLM_API_KEY` | provider credential | empty | +| `LLM_CHAT_MODEL` | chat/completion model | `gpt-4.1-mini` | +| `LLM_EMBEDDING_MODEL` | embedding model | `text-embedding-3-small` | +| `LLM_EMBEDDING_DIMENSIONS` | expected vector length | `1536` | +| `LLM_TIMEOUT_SECONDS` | request timeout | `60` | +| `LLM_MAX_RETRIES` | retry count | `4` | +| `LLM_RETRY_INITIAL_SECONDS` | initial backoff | `1` | +| `LLM_RETRY_MAX_SECONDS` | maximum backoff | `30` | + +Retries cover 429, 500/502/503/504, connection failures, and timeouts. +Numeric `Retry-After` is honored. Authentication and validation failures are +not retried. + +## Processing and retrieval + +| Variable | Purpose | Default | +|---|---|---:| +| `CHUNK_SIZE` | chunk target size | `1200` | +| `CHUNK_OVERLAP` | chunk overlap | `150` | +| `COMMUNITY_RESOLUTION` | NetworkX community resolution | `1.0` | +| `RETRIEVAL_TOP_K` | vector result count | `8` | +| `RETRIEVAL_GRAPH_HOPS` | graph expansion depth | `2` | +| `RETRIEVAL_GRAPH_LIMIT` | graph expansion cap | `50` | +| `RETRIEVAL_HNSW_EF_SEARCH` | PgVector search breadth | `100` | + +`CHUNK_OVERLAP` must be smaller than `CHUNK_SIZE`. Embedding dimensions must +match the provider output and the PgVector index dimension. + +## Security and worker + +| Variable | Purpose | +|---|---| +| `INTERNAL_JWT_SECRET` | HS256 verification secret | +| `INTERNAL_JWT_ISSUER` | expected JWT issuer | +| `INTERNAL_JWT_AUDIENCE` | expected JWT audience | +| `INTERNAL_FEATURE_NAME` | feature key containing permission bits | +| `LOG_LEVEL` | structured log level | +| `WORKER_POLL_SECONDS` | idle worker polling interval | +| `WORKER_STALE_RUN_SECONDS` | stale-run recovery threshold | + +Change every development secret before shared or production use. Prefer workload +identity, mTLS, or asymmetric/JWKS verification in production. + diff --git a/docs/data-model.md b/docs/data-model.md new file mode 100644 index 0000000..682fe92 --- /dev/null +++ b/docs/data-model.md @@ -0,0 +1,38 @@ +# Data model and checkpoint ownership + +The initial Alembic revision creates the SQLAlchemy metadata and the PgVector +extension. The models live in [`src/graphrag_platform/models.py`](../src/graphrag_platform/models.py). + +## Pipeline data + +| Table/model | Owner stage | Meaning | +|---|---:|---| +| `raw_documents` | 1 | original source payload and provenance | +| `documents` | 1 | normalized text and source identity | +| `chunks` | 2 | cleaned, overlapping retrieval units | +| `knowledge_models` | 3 | configured entity/relationship vocabulary and templates | +| `extractions` | 4 | validated LLM payload per chunk | +| `resolved_entities` | 5 | canonical entities and aliases | +| `resolved_relationships` | 5 | relationships linked to canonical keys | +| `nodes` | 6 | graph nodes | +| `edges` | 6 | graph edges | +| `communities` | 7 | detected groups and summaries | +| `search_items` | 8 | text, metadata, source URL, and embedding | + +## Run tracking + +| Table/model | Purpose | +|---|---| +| `pipeline_runs` | requested range, status, attempt, caller, correlation, and errors | +| `stage_runs` | per-stage status, input/output, timestamps, and error details | + +Stage data is corpus-scoped. Clear or rebuild from the first invalid stage +through downstream derived data; do not manually delete unrelated corpora. + +## Indexing + +PostgreSQL creates a dimension-specific HNSW cosine index on retrieval items. +SQLite stores vectors as JSON and uses in-process cosine similarity. The latter +is useful for tests but does not validate production query plans or PgVector +behavior. + diff --git a/docs/diagrams/c4-code.drawio b/docs/diagrams/c4-code.drawio new file mode 100644 index 0000000..93d9129 --- /dev/null +++ b/docs/diagrams/c4-code.drawio @@ -0,0 +1 @@ + diff --git a/docs/diagrams/c4-code.mmd b/docs/diagrams/c4-code.mmd new file mode 100644 index 0000000..a25bccd --- /dev/null +++ b/docs/diagrams/c4-code.mmd @@ -0,0 +1,20 @@ +flowchart LR + s1("01 normalize
raw + documents") + s2("02 clean/chunk
chunks") + s3("03 model
knowledge_models") + s4("04 extract
extractions") + s5("05 reconcile
resolved data") + s6("06 graph
nodes + edges") + s7("07 communities
summaries") + s8("08 index
search_items + HNSW") + s9("09 retrieve
vector + graph context") + s10("10 answer
cited response") + + s1 --> s2 --> s3 --> s4 --> s5 --> s6 --> s7 --> s8 --> s9 --> s10 + + classDef preparation fill:#dae8fc,stroke:#6c8ebf,color:#000; + classDef indexing fill:#d5e8d4,stroke:#82b366,color:#000; + classDef querying fill:#fff2cc,stroke:#d6b656,color:#000; + class s1,s2,s3,s4,s5 preparation; + class s6,s7,s8 indexing; + class s9,s10 querying; diff --git a/docs/diagrams/c4-code.pdf b/docs/diagrams/c4-code.pdf new file mode 100644 index 0000000..5797b6d Binary files /dev/null and b/docs/diagrams/c4-code.pdf differ diff --git a/docs/diagrams/c4-component.drawio b/docs/diagrams/c4-component.drawio new file mode 100644 index 0000000..35be60b --- /dev/null +++ b/docs/diagrams/c4-component.drawio @@ -0,0 +1 @@ + diff --git a/docs/diagrams/c4-component.mmd b/docs/diagrams/c4-component.mmd new file mode 100644 index 0000000..b0d8484 --- /dev/null +++ b/docs/diagrams/c4-component.mmd @@ -0,0 +1,18 @@ +flowchart LR + api("api.py
Routes and request models") + auth("auth.py
JWT and permission checks") + orch("orchestrator.py
Durable runs and locks") + runtime("runtime.py + stage_loader.py
Context and stage dispatch") + storage("storage.py + models.py
SQLAlchemy and PgVector") + llm("llm.py
Provider client and retries") + + api -->|"authorize"| auth + api -->|"queue"| orch + orch -->|"dispatch"| runtime + runtime -->|"persist"| storage + runtime -->|"complete"| llm + + classDef application fill:#dae8fc,stroke:#6c8ebf,color:#000; + classDef dependency fill:#d5e8d4,stroke:#82b366,color:#000; + class api,auth,orch,runtime application; + class storage,llm dependency; diff --git a/docs/diagrams/c4-component.pdf b/docs/diagrams/c4-component.pdf new file mode 100644 index 0000000..442ad18 Binary files /dev/null and b/docs/diagrams/c4-component.pdf differ diff --git a/docs/diagrams/c4-container.drawio b/docs/diagrams/c4-container.drawio new file mode 100644 index 0000000..bdeadab --- /dev/null +++ b/docs/diagrams/c4-container.drawio @@ -0,0 +1 @@ + diff --git a/docs/diagrams/c4-container.mmd b/docs/diagrams/c4-container.mmd new file mode 100644 index 0000000..2317a01 --- /dev/null +++ b/docs/diagrams/c4-container.mmd @@ -0,0 +1,19 @@ +flowchart LR + api("FastAPI API
Internal query, answer, and job routes") + worker("Worker
Claims and executes durable runs") + db[("PostgreSQL + PgVector
Checkpoints, graph, vectors, HNSW")] + stages("Numbered stage modules
Normalize through answer") + llm("OpenAI-compatible provider
Chat and embeddings") + + api -->|"dispatch"| worker + worker -->|"execute"| stages + stages -->|"persist/retrieve"| db + stages -->|"prompts/vectors"| llm + api -->|"status and query"| db + + classDef application fill:#dae8fc,stroke:#6c8ebf,color:#000; + classDef storage fill:#d5e8d4,stroke:#82b366,color:#000; + classDef external fill:#fff2cc,stroke:#d6b656,color:#000; + class api,worker,stages application; + class db storage; + class llm external; diff --git a/docs/diagrams/c4-container.pdf b/docs/diagrams/c4-container.pdf new file mode 100644 index 0000000..0fa64b7 Binary files /dev/null and b/docs/diagrams/c4-container.pdf differ diff --git a/docs/diagrams/c4-context.drawio b/docs/diagrams/c4-context.drawio new file mode 100644 index 0000000..30e168c --- /dev/null +++ b/docs/diagrams/c4-context.drawio @@ -0,0 +1 @@ + diff --git a/docs/diagrams/c4-context.mmd b/docs/diagrams/c4-context.mmd new file mode 100644 index 0000000..469d371 --- /dev/null +++ b/docs/diagrams/c4-context.mmd @@ -0,0 +1,14 @@ +flowchart LR + kg("KnowledgeGraph GraphRAG") + mcp("Web-Access MCP + Data-Store MCP
External source collection") + bff("Existing BFF
Authentication and policy") + llm("OpenAI-compatible provider
Chat and embeddings") + + mcp -->|"Versioned source handoff"| kg + bff -->|"Signed internal JWT
and API calls"| kg + kg -->|"Prompts and embeddings"| llm + + classDef application fill:#dae8fc,stroke:#6c8ebf,color:#000; + classDef external fill:#fff2cc,stroke:#d6b656,color:#000; + class kg application; + class mcp,bff,llm external; diff --git a/docs/diagrams/c4-context.pdf b/docs/diagrams/c4-context.pdf new file mode 100644 index 0000000..dcf0460 Binary files /dev/null and b/docs/diagrams/c4-context.pdf differ diff --git a/docs/module-reference.md b/docs/module-reference.md new file mode 100644 index 0000000..7336e68 --- /dev/null +++ b/docs/module-reference.md @@ -0,0 +1,46 @@ +# Module reference + +## Platform package + +| Module | Responsibility | +|---|---| +| `config.py` | typed settings, environment parsing, model definition | +| `models.py` | SQLAlchemy entities and checkpoint schema | +| `storage.py` | engine/session, schema checks, persistence, vector index | +| `sources.py` | strict JSONL and PostgreSQL source readers | +| `llm.py` | OpenAI-compatible completion and embedding client | +| `runtime.py` | `StageContext` construction and initialization | +| `stage_loader.py` | numbered stage module lookup | +| `pipeline.py` | compatibility facade for stage methods | +| `orchestrator.py` | durable runs, claiming, resume, stale recovery, locks | +| `auth.py` | JWT verification and permission masks | +| `api.py` | FastAPI app, request models, routes, authorization | +| `worker.py` | polling worker and one-run execution | +| `cli.py` | stage and durable-run command line entry points | +| `observability.py` | structured JSON logging | + +## Numbered stages + +| Module | Main responsibility | +|---|---| +| `01_normalize_content.py` | validate handoff, extract text, persist raw/normalized documents | +| `02_clean_content.py` | clean, deduplicate, and chunk | +| `03_define_knowledge_model.py` | persist entity/relationship vocabulary and templates | +| `04_extract_knowledge.py` | call LLM, validate extraction, persist per-chunk results | +| `05_reconcile_knowledge.py` | canonicalize names and resolve cross-chunk relationships | +| `06_build_knowledge_graph.py` | materialize nodes and edges | +| `07_build_communities.py` | detect communities and summarize them | +| `08_build_retrieval_index.py` | embed chunks/graph/community items and build HNSW | +| `09_query_graphrag.py` | vector retrieval, graph expansion, source resolution | +| `10_generate_answer.py` | generate answer with source citations | + +Each numbered module exposes a `run` function and command-line adapter. Stages +1-5 also expose focused validation/transformation helpers; stage 9 exposes +retrieval helpers such as cosine ranking and graph expansion. + +## Entry points + +- `knowledgegraph` -> `graphrag_platform.cli:main_all` +- `knowledgegraph-worker` -> `graphrag_platform.worker:main` +- `uvicorn graphrag_platform.api:app` + diff --git a/docs/operations.md b/docs/operations.md new file mode 100644 index 0000000..d1316d9 --- /dev/null +++ b/docs/operations.md @@ -0,0 +1,76 @@ +# Operations and troubleshooting + +## Health and readiness + +- `/internal/knowledge-graph/health` checks process liveness. +- `/internal/knowledge-graph/ready` checks database and PgVector readiness. +- `podman compose ps` shows service and health state. +- `podman compose logs --follow api worker` streams structured JSON logs. + +Logs include event names, run IDs, corpus/stage context, attempts, status, and +error details. Secrets are not intended to be logged. + +## Common failures + +### Migration does not complete + +Check PostgreSQL health, credentials, and `DATABASE_URL`. Confirm the PgVector +image is used and inspect: + +```powershell +podman compose logs migrate postgres +alembic current +``` + +Do not enable automatic schema creation in a shared environment. + +### Source validation fails + +Check schema version, required fields, ISO timestamp offset, unique +`external_id`, and corpus. The error identifies the source file and line. + +### LLM requests fail + +Verify `LLM_BASE_URL`, key, model names, and embedding dimensions. 429 and +transient server/network errors retry; authentication and validation errors fail +immediately. Check provider quota and `Retry-After` behavior before increasing +retry settings. + +### Vector index or retrieval fails + +Confirm the `vector` extension exists, the provider returns exactly +`LLM_EMBEDDING_DIMENSIONS` values, and stage 8 completed for the selected +corpus. SQLite cannot validate PgVector behavior. + +### Job remains queued + +Check worker logs and the corpus advisory lock. Another build may be active. +The worker requeues stale runs after `WORKER_STALE_RUN_SECONDS`; set this above +the maximum expected stage duration. + +### Job fails after partial progress + +Read the job and stage error, repair the source/configuration, then retry. Retry +resumes after the last successful stage. Use a reset rebuild if partial side +effects or changed model settings make the checkpoint unsafe. + +## Known limitations + +- Only source schema `1.0` is accepted. +- `ANALYTICS_DATABASE_URL` is configured but unused by the current pipeline. +- Source table names are simple unqualified names. +- Stage-level recovery is not an all-stage transaction. +- Name normalization can raise on conflicting entity types. +- LLM shape validation does not establish factual correctness. +- Answer citations are prompt-enforced, not independently checked. +- No observed API rate limiting or pagination middleware. +- The migration set currently contains one initial revision. + +## Operational safety + +- Use read-only credentials for the external source database. +- Keep API and worker on private network paths behind the BFF. +- Rotate all development secrets before deployment. +- Back up PostgreSQL before destructive resets or migration changes. +- Treat `podman compose down --volumes` as destructive. + diff --git a/docs/overview.md b/docs/overview.md new file mode 100644 index 0000000..de674ec --- /dev/null +++ b/docs/overview.md @@ -0,0 +1,68 @@ +# System overview + +## Purpose + +KnowledgeGraph GraphRAG turns externally collected documents into a +source-grounded knowledge graph and vector retrieval index. It then combines +semantic retrieval with bounded graph expansion to produce context and cited +answers. + +The collection boundary is deliberate: Web-Access MCP and Data-Store MCP +discover and persist source material. This repository starts at their versioned +handoff. + +## End-to-end flow + +```text +external source handoff + -> 1 normalize + -> 2 clean, deduplicate, chunk + -> 3 persist knowledge model + -> 4 extract entities and relationships + -> 5 reconcile aliases and duplicates + -> 6 materialize nodes and edges + -> 7 detect and summarize communities + -> 8 embed and index + -> 9 retrieve vector + graph + source context + -> 10 generate cited answer +``` + +Stages 1-8 persist durable checkpoints in PostgreSQL. A later stage can be +re-run without repeating valid upstream work. A durable run records stage +selection, attempts, configuration fingerprint, status, timestamps, and errors. + +## Runtime boundary + +The deployment contains: + +1. PostgreSQL 16 with the PgVector extension. +2. An Alembic migration job. +3. An internal FastAPI service. +4. A PostgreSQL-backed worker. +5. An existing BFF that authenticates callers and enforces feature permissions. +6. An OpenAI-compatible chat and embedding provider. + +The API is an internal service and is not intended for direct customer access. + +## Important design decisions + +- PostgreSQL is the canonical checkpoint store. +- PgVector performs server-side cosine search using a dimension-specific HNSW + index. +- SQLite is a local-test fallback with JSON embeddings and in-process search. +- LLM calls happen outside long database transactions; validated chunk results + are committed independently. +- A PostgreSQL advisory lock permits one active build per corpus. +- The worker can recover stale runs and resume after the last successful stage. +- The BFF supplies a short-lived signed internal JWT; unsigned permission + headers are not trusted. + +## Non-goals and boundaries + +- Crawling and source discovery are not implemented here. +- The API does not provide customer-facing authentication or authorization. +- Query and answer are not queued worker jobs. +- Citation correctness is requested through the answer prompt, not independently + verified. +- Entity reconciliation is normalized-name based, not semantic entity matching. + diff --git a/docs/runbook.md b/docs/runbook.md new file mode 100644 index 0000000..c7e025c --- /dev/null +++ b/docs/runbook.md @@ -0,0 +1,131 @@ +# Step-by-step runbook + +Commands use PowerShell on Windows and assume the repository root is current. + +## 1. Prepare tools and environment + +Prerequisites: Python 3.11+, Podman 5+, a running `podman machine`, and an +OpenAI-compatible chat/embedding endpoint. + +```powershell +Copy-Item .env.example .env +py -3.12 -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +python -m pip install -e ".[test]" +``` + +Edit `.env`: set `POSTGRES_PASSWORD`, matching `DATABASE_URL` credentials, +`LLM_API_KEY`, model names/dimensions, and a random +`INTERNAL_JWT_SECRET` of at least 32 bytes. + +## 2. Start the stack + +```powershell +podman machine start +podman compose up --detach --build +podman compose ps +Invoke-RestMethod http://127.0.0.1:8080/internal/knowledge-graph/health +Invoke-RestMethod http://127.0.0.1:8080/internal/knowledge-graph/ready +``` + +Compose starts PostgreSQL, applies migrations, and then starts API and worker. +The migration service must complete before the application services start. + +## 3. Supply a source handoff + +Place a UTF-8 JSONL file in `.\artifacts`, or configure a PostgreSQL source +table. Validate it against the [source contract](./source-contract.md). For a +file handoff, use `.\artifacts\mcp-documents.jsonl`. + +## 4. Run a durable build + +```powershell +knowledgegraph run ` + --corpus product-knowledge ` + --from-stage 1 ` + --through-stage 8 ` + --input-jsonl .\artifacts\mcp-documents.jsonl ` + --model-file .\src\templates\knowledge_model.json ` + --requested-by local-developer +``` + +Watch progress: + +```powershell +podman compose logs --follow api worker +``` + +## 5. Query and answer + +For direct local stage execution: + +```powershell +python .\src\09_query_graphrag.py ` + --corpus product-knowledge ` + --question "Who created Atlas?" ` + --top-k 8 + +python .\src\10_generate_answer.py ` + --corpus product-knowledge ` + --question "Who created Atlas?" +``` + +For the API, obtain a BFF-issued JWT and call `/query` or `/answer` as described +in the [API reference](./api.md). + +## 6. Rebuild downstream data + +Use `--from-stage` to avoid repeating valid upstream work. Use `--reset` when +the selected stage or any downstream data is inconsistent: + +```powershell +knowledgegraph run ` + --corpus product-knowledge ` + --from-stage 4 ` + --through-stage 8 ` + --reset ` + --requested-by rebuild +``` + +Do not run two builds for the same corpus concurrently; the PostgreSQL advisory +lock will defer one, but coordinating callers is clearer. + +## 7. Apply or inspect migrations + +```powershell +alembic current +alembic history +alembic upgrade head +``` + +`AUTO_CREATE_SCHEMA=true` is for isolated tests only. Review generated +migrations before applying them. + +## 8. Run tests + +```powershell +pytest --ignore=.\tests\integration\test_pgvector_postgres.py +``` + +For the real PgVector integration: + +```powershell +$env:TEST_POSTGRES_URL = "postgresql+psycopg://knowledgegraph_app:password@localhost:5432/knowledgegraph" +pytest .\tests\integration\test_pgvector_postgres.py +Remove-Item Env:TEST_POSTGRES_URL +``` + +## 9. Stop or reset local services + +```powershell +podman compose down +``` + +This retains the database volume. To delete all local PostgreSQL data: + +```powershell +podman compose down --volumes +``` + +Volume removal is destructive. diff --git a/docs/source-contract.md b/docs/source-contract.md new file mode 100644 index 0000000..1c9c60b --- /dev/null +++ b/docs/source-contract.md @@ -0,0 +1,72 @@ +# External source contract + +## JSON Lines + +Each UTF-8 line must be a JSON object with schema version `1.0`. + +```json +{ + "schema_version": "1.0", + "external_id": "stable-source-record-id", + "corpus": "product-knowledge", + "url": "https://docs.example.com/article", + "body": "
Article text
", + "content_type": "text/html", + "status_code": 200, + "retrieved_at": "2026-10-13T18:00:00Z", + "metadata": { + "collector": "Web-Access", + "stored_by": "Data-Store" + } +} +``` + +Required fields: + +- `schema_version`: exactly `1.0`; +- `external_id`: stable and unique within a corpus; +- `url`; +- `body`; +- `retrieved_at`: ISO 8601 timestamp with an offset. + +Defaulted fields: + +- `content_type`: `text/html`; +- `status_code`: `200`; +- `metadata`: `{}`. + +`corpus`, when provided, must equal the selected runtime corpus. Unknown fields +and unsupported schema versions are rejected. Validation errors include the +file and line number. See the canonical template at +[`src/templates/mcp_documents.jsonl`](../src/templates/mcp_documents.jsonl) and +the acceptance fixture at [`tests/fixtures/external_documents.jsonl`](../tests/fixtures/external_documents.jsonl). + +## PostgreSQL source table + +Set `SOURCE_DATABASE_URL` and `SOURCE_TABLE_NAME` to use a PostgreSQL handoff. +The table must contain: + +```sql +CREATE TABLE external_documents ( + external_id text PRIMARY KEY, + schema_version text NOT NULL DEFAULT '1.0', + corpus text, + url text NOT NULL, + body text NOT NULL, + content_type text NOT NULL DEFAULT 'text/html', + status_code integer NOT NULL DEFAULT 200, + retrieved_at timestamptz NOT NULL, + metadata jsonb NOT NULL DEFAULT '{}'::jsonb +); +``` + +Additional columns are ignored. Give the application account read-only access. +Use either `--input-jsonl` or `SOURCE_DATABASE_URL`, not both. + +## Normalization behavior + +Stage 1 stores raw provenance and a normalized document. HTML is extracted with +Trafilatura; non-HTML content is stripped directly. Stage 2 normalizes +whitespace, detects duplicate content by SHA-256 hash, and creates overlapping +chunks. + diff --git a/docs/testing.md b/docs/testing.md new file mode 100644 index 0000000..c0da51e --- /dev/null +++ b/docs/testing.md @@ -0,0 +1,47 @@ +# Testing and validation + +## Test layers + +| Layer | Coverage | +|---|---| +| Unit | settings, source validation, chunk constraints, reconciliation, provider retry policy | +| Integration | all ten stages, API/worker jobs, repair/retry/resume | +| PostgreSQL integration | migrations, PgVector extension, HNSW index, graph retrieval, advisory lock | +| Performance | configurable vector query load against real PostgreSQL | + +The deterministic `FakeLLM` in `tests/conftest.py` keeps portable tests +repeatable. It is not a quality or latency substitute for a production model. + +## Local validation + +```powershell +pytest --ignore=.\tests\integration\test_pgvector_postgres.py +python -m compileall -q src tests migrations +python -m pip check +``` + +The CI workflow also validates Compose configuration and builds the +`Containerfile` image. + +## PostgreSQL integration + +Set `TEST_POSTGRES_URL` to a PgVector-enabled database. The migration test +resets the schema, creates the external source table, runs stages 1-9, checks +the vector extension and HNSW definition, and verifies advisory-lock behavior. + +## Performance benchmark + +The benchmark is skipped unless both `TEST_POSTGRES_URL` and +`RUN_PERFORMANCE_TESTS=1` are set: + +```powershell +$env:TEST_POSTGRES_URL = "postgresql+psycopg://knowledgegraph_app:password@localhost:5432/knowledgegraph" +$env:RUN_PERFORMANCE_TESTS = "1" +$env:PERF_VECTOR_ITEM_COUNT = "10000" +$env:PERF_MAX_QUERY_SECONDS = "2.0" +pytest .\tests\performance\test_vector_query_load.py -m performance +``` + +The benchmark inserts deterministic three-dimensional vectors and measures +stage 9 retrieval. Results depend on hardware, database configuration, and load. + diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..8cc2451 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +from graphrag_platform.config import Settings +from graphrag_platform.models import Base + +config = context.config +if config.config_file_name: + fileConfig(config.config_file_name) + +config.set_main_option("sqlalchemy.url", Settings().database_url) +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + context.configure( + url=config.get_main_option("sqlalchemy.url"), + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..590f5b3 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/20261013_0001_initial.py b/migrations/versions/20261013_0001_initial.py new file mode 100644 index 0000000..61eafbb --- /dev/null +++ b/migrations/versions/20261013_0001_initial.py @@ -0,0 +1,25 @@ +"""Create the initial KnowledgeGraph schema. + +Revision ID: 20261013_0001 +Revises: +Create Date: 2026-10-13 +""" + +from alembic import op + +from graphrag_platform.models import Base + +revision = "20261013_0001" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + connection = op.get_bind() + connection.exec_driver_sql("CREATE EXTENSION IF NOT EXISTS vector") + Base.metadata.create_all(bind=connection) + + +def downgrade() -> None: + Base.metadata.drop_all(bind=op.get_bind()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..15147dd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,60 @@ +[build-system] +requires = ["hatchling>=1.25"] +build-backend = "hatchling.build" + +[project] +name = "knowledgegraph-graphrag" +version = "0.1.0" +description = "Production-oriented KnowledgeGraph ingestion, indexing, retrieval, and answer pipeline" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "alembic>=1.13,<2", + "fastapi>=0.115,<1", + "httpx>=0.27,<1", + "networkx>=3.3,<4", + "pgvector>=0.3,<1", + "PyJWT>=2.9,<3", + "pydantic>=2.8,<3", + "pydantic-settings>=2.4,<3", + "psycopg[binary]>=3.2,<4", + "sqlalchemy>=2.0,<3", + "trafilatura>=1.12,<3", + "uvicorn[standard]>=0.32,<1", +] + +[project.optional-dependencies] +test = [ + "pytest>=8.3,<9", + "pytest-cov>=5,<7", +] + +[project.scripts] +knowledgegraph = "graphrag_platform.cli:main_all" +knowledgegraph-worker = "graphrag_platform.worker:main" + +[tool.hatch.build.targets.wheel] +packages = [ + "src/graphrag_platform", + "src/01_normalize_content.py", + "src/02_clean_content.py", + "src/03_define_knowledge_model.py", + "src/04_extract_knowledge.py", + "src/05_reconcile_knowledge.py", + "src/06_build_knowledge_graph.py", + "src/07_build_communities.py", + "src/08_build_retrieval_index.py", + "src/09_query_graphrag.py", + "src/10_generate_answer.py", +] + +[tool.pytest.ini_options] +addopts = "-ra" +testpaths = ["tests"] +markers = [ + "performance: opt-in performance and load checks", +] + +[tool.coverage.run] +source = ["graphrag_platform"] +branch = true \ No newline at end of file diff --git a/src/01_normalize_content.py b/src/01_normalize_content.py new file mode 100644 index 0000000..440c7ea --- /dev/null +++ b/src/01_normalize_content.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import trafilatura + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import Document, RawDocument +from graphrag_platform.runtime import StageContext +from graphrag_platform.sources import SourceRecord, source_reader + + +def content_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def extract_document(record: SourceRecord) -> dict[str, object]: + if "html" not in record.content_type.casefold(): + text = record.body.strip() + if not text: + raise ValueError(f"source content is empty for {record.url}") + return { + "title": record.metadata.get("title"), + "author": record.metadata.get("author"), + "date": record.metadata.get("published_at"), + "hostname": None, + "sitename": None, + "text": text, + } + extracted = trafilatura.extract( + record.body, + url=str(record.url), + output_format="json", + with_metadata=True, + include_comments=False, + include_tables=True, + favor_precision=True, + ) + if not extracted: + raise ValueError(f"could not extract article content from {record.url}") + payload = json.loads(extracted) + if not str(payload.get("text") or "").strip(): + raise ValueError(f"extracted content is empty for {record.url}") + return payload + + +def run( + context: StageContext, + *, + input_jsonl: Path | None = None, +) -> dict[str, int]: + context.initialize() + reader = source_reader( + input_jsonl=input_jsonl, + source_database_url=context.settings.source_database_url, + source_table_name=context.settings.source_table_name, + ) + normalized = 0 + with context.storage.session() as session: + for record in reader.records(): + if record.corpus and record.corpus != context.settings.corpus_name: + raise ValueError( + f"record {record.external_id} targets corpus {record.corpus}, " + f"not {context.settings.corpus_name}" + ) + payload = extract_document(record) + text = str(payload["text"]).strip() + metadata = dict(record.metadata) + metadata.update( + { + "external_id": record.external_id, + "source_schema_version": record.schema_version, + "retrieved_at": record.retrieved_at.isoformat(), + "hostname": payload.get("hostname"), + "sitename": payload.get("sitename"), + } + ) + raw = context.storage.upsert( + session, + RawDocument, + { + "corpus": context.settings.corpus_name, + "external_id": record.external_id, + }, + { + "source_schema_version": record.schema_version, + "url": str(record.url), + "status_code": record.status_code, + "content_type": record.content_type, + "body": record.body, + "metadata_json": record.metadata, + "retrieved_at": record.retrieved_at, + }, + ) + context.storage.upsert( + session, + Document, + {"raw_document_id": raw.id}, + { + "corpus": context.settings.corpus_name, + "url": str(record.url), + "title": payload.get("title"), + "author": payload.get("author"), + "published_at": payload.get("date"), + "text": text, + "content_hash": content_hash(text), + "metadata_json": metadata, + "is_duplicate": False, + "canonical_document_id": None, + }, + ) + normalized += 1 + if normalized == 0: + raise ValueError("source handoff contained no records") + return {"normalized": normalized} + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + return run(context, input_jsonl=args.input_jsonl) + + +if __name__ == "__main__": + stage_main(1, run_from_args) diff --git a/src/02_clean_content.py b/src/02_clean_content.py new file mode 100644 index 0000000..cd3bb3f --- /dev/null +++ b/src/02_clean_content.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import argparse +import hashlib +import re + +from sqlalchemy import delete, select + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import Chunk, Document +from graphrag_platform.runtime import StageContext + +_SPACE_RE = re.compile(r"[ \t]+") +_PARAGRAPH_RE = re.compile(r"\n{3,}") + + +def content_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def clean_text(text: str) -> str: + lines = [_SPACE_RE.sub(" ", line).strip() for line in text.splitlines()] + return _PARAGRAPH_RE.sub("\n\n", "\n".join(lines)).strip() + + +def chunk_text(text: str, *, size: int, overlap: int) -> list[str]: + if overlap >= size: + raise ValueError("CHUNK_OVERLAP must be smaller than CHUNK_SIZE") + chunks: list[str] = [] + start = 0 + while start < len(text): + end = min(start + size, len(text)) + if end < len(text): + break_at = text.rfind(" ", start + size // 2, end) + if break_at > start: + end = break_at + chunk = text[start:end].strip() + if chunk: + chunks.append(chunk) + if end >= len(text): + break + start = end - overlap + return chunks + + +def run(context: StageContext) -> dict[str, int]: + context.initialize() + chunk_count = 0 + duplicate_count = 0 + with context.storage.session() as session: + documents = session.scalars( + select(Document) + .where(Document.corpus == context.settings.corpus_name) + .order_by(Document.id) + ).all() + if not documents: + raise ValueError("stage 2 requires normalized documents") + session.execute( + delete(Chunk).where(Chunk.corpus == context.settings.corpus_name) + ) + hashes: dict[str, int] = {} + for document in documents: + text = clean_text(document.text) + digest = content_hash(text) + document.text = text + document.content_hash = digest + if digest in hashes: + document.is_duplicate = True + document.canonical_document_id = hashes[digest] + duplicate_count += 1 + continue + document.is_duplicate = False + document.canonical_document_id = None + hashes[digest] = document.id + for ordinal, text_chunk in enumerate( + chunk_text( + text, + size=context.settings.chunk_size, + overlap=context.settings.chunk_overlap, + ) + ): + session.add( + Chunk( + document_id=document.id, + corpus=context.settings.corpus_name, + ordinal=ordinal, + text=text_chunk, + content_hash=content_hash(text_chunk), + ) + ) + chunk_count += 1 + return {"chunks": chunk_count, "duplicates": duplicate_count} + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + del args + return run(context) + + +if __name__ == "__main__": + stage_main(2, run_from_args) diff --git a/src/03_define_knowledge_model.py b/src/03_define_knowledge_model.py new file mode 100644 index 0000000..2b8c25e --- /dev/null +++ b/src/03_define_knowledge_model.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import KnowledgeModel +from graphrag_platform.runtime import StageContext + + +def validate_model_definition(definition: dict[str, Any]) -> None: + for key in ("entity_types", "relationship_types"): + value = definition.get(key) + if not isinstance(value, list) or not value or not all( + isinstance(item, str) and item.strip() for item in value + ): + raise ValueError(f"{key} must be a non-empty array of strings") + for key in ("node_template", "edge_template"): + if not isinstance(definition.get(key), dict): + raise ValueError(f"{key} must be a JSON object") + + +def run( + context: StageContext, *, model_file: Path | None = None +) -> dict[str, int]: + context.initialize() + definition = context.settings.model_definition() + if model_file is not None: + loaded = json.loads(model_file.read_text(encoding="utf-8")) + if not isinstance(loaded, dict): + raise ValueError("knowledge model file must contain a JSON object") + definition.update(loaded) + validate_model_definition(definition) + with context.storage.session() as session: + context.storage.upsert( + session, + KnowledgeModel, + {"corpus": context.settings.corpus_name}, + {"definition_json": definition}, + ) + return { + "entity_types": len(definition["entity_types"]), + "relationship_types": len(definition["relationship_types"]), + } + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + return run(context, model_file=args.model_file) + + +if __name__ == "__main__": + stage_main(3, run_from_args) diff --git a/src/04_extract_knowledge.py b/src/04_extract_knowledge.py new file mode 100644 index 0000000..c98acb6 --- /dev/null +++ b/src/04_extract_knowledge.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import argparse +import json +from typing import Any + +from sqlalchemy import select + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import Chunk, Extraction, KnowledgeModel +from graphrag_platform.runtime import StageContext + + +def validate_extraction( + payload: dict[str, Any], definition: dict[str, Any] +) -> None: + entities = payload.get("entities") + relationships = payload.get("relationships") + if not isinstance(entities, list) or not isinstance(relationships, list): + raise ValueError("extraction requires entities and relationships arrays") + entity_types = set(definition["entity_types"]) + relationship_types = set(definition["relationship_types"]) + for entity in entities: + if not isinstance(entity, dict): + raise ValueError("each entity must be an object") + if not str(entity.get("canonical_name", "")).strip(): + raise ValueError("entity canonical_name must not be empty") + if entity.get("entity_type") not in entity_types: + raise ValueError(f"unsupported entity type: {entity.get('entity_type')}") + if not isinstance(entity.get("aliases", []), list): + raise ValueError("entity aliases must be an array") + for relationship in relationships: + if not isinstance(relationship, dict): + raise ValueError("each relationship must be an object") + if not str(relationship.get("source", "")).strip() or not str( + relationship.get("target", "") + ).strip(): + raise ValueError("relationship source and target must not be empty") + if relationship.get("relationship_type") not in relationship_types: + raise ValueError( + "unsupported relationship type: " + f"{relationship.get('relationship_type')}" + ) + + +def run(context: StageContext) -> dict[str, int]: + context.initialize() + with context.storage.session() as session: + model = session.get(KnowledgeModel, context.settings.corpus_name) + chunks = session.scalars( + select(Chunk).where(Chunk.corpus == context.settings.corpus_name) + ).all() + if model is None or not chunks: + raise ValueError("stage 4 requires a knowledge model and cleaned chunks") + definition = dict(model.definition_json) + chunk_inputs = [(chunk.id, chunk.text) for chunk in chunks] + + system = ( + "You extract a source-grounded knowledge graph. Return one JSON object " + 'with arrays named "entities" and "relationships". Never infer unsupported facts.' + ) + extracted_count = 0 + for chunk_id, chunk_text in chunk_inputs: + prompt = json.dumps( + { + "knowledge_model": definition, + "source_chunk_id": chunk_id, + "text": chunk_text, + }, + ensure_ascii=False, + ) + payload = context.llm.complete_json(system, prompt) + validate_extraction(payload, definition) + with context.storage.session() as session: + context.storage.upsert( + session, + Extraction, + {"chunk_id": chunk_id}, + { + "corpus": context.settings.corpus_name, + "payload_json": payload, + "model_name": context.llm.chat_model, + }, + ) + extracted_count += 1 + return {"extracted_chunks": extracted_count} + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + del args + return run(context) + + +if __name__ == "__main__": + stage_main(4, run_from_args) diff --git a/src/05_reconcile_knowledge.py b/src/05_reconcile_knowledge.py new file mode 100644 index 0000000..2dba4c9 --- /dev/null +++ b/src/05_reconcile_knowledge.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import argparse +import re +from typing import Any + +from sqlalchemy import delete, select + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import ( + Extraction, + ResolvedEntity, + ResolvedRelationship, +) +from graphrag_platform.runtime import StageContext + +_KEY_RE = re.compile(r"[^\w]+", re.UNICODE) + + +def canonical_key(name: str) -> str: + return _KEY_RE.sub(" ", name.casefold()).strip() + + +def merge_text(current: str, incoming: str) -> str: + current = current.strip() + incoming = incoming.strip() + if not incoming or incoming.casefold() in current.casefold(): + return current + if not current: + return incoming + return f"{current}\n{incoming}" + + +def run(context: StageContext) -> dict[str, int]: + context.initialize() + entity_count = 0 + relationship_count = 0 + with context.storage.session() as session: + extractions = session.scalars( + select(Extraction).where( + Extraction.corpus == context.settings.corpus_name + ) + ).all() + if not extractions: + raise ValueError("stage 5 requires extracted knowledge") + session.execute( + delete(ResolvedRelationship).where( + ResolvedRelationship.corpus == context.settings.corpus_name + ) + ) + session.execute( + delete(ResolvedEntity).where( + ResolvedEntity.corpus == context.settings.corpus_name + ) + ) + entity_rows: dict[str, dict[str, Any]] = {} + relation_rows: dict[tuple[str, str, str], dict[str, Any]] = {} + pending_relationships: list[tuple[int, dict[str, Any]]] = [] + for extraction in extractions: + payload = extraction.payload_json + for entity in payload["entities"]: + name = entity["canonical_name"].strip() + key = canonical_key(name) + row = entity_rows.get(key) + if row is None: + row = { + "canonical_name": name, + "entity_type": entity["entity_type"], + "description": "", + "aliases": set(), + "chunk_ids": set(), + } + entity_rows[key] = row + elif row["entity_type"] != entity["entity_type"]: + raise ValueError( + f"conflicting entity types for {name}: " + f"{row['entity_type']} and {entity['entity_type']}" + ) + row["description"] = merge_text( + row["description"], entity.get("description", "") + ) + row["aliases"].update(entity.get("aliases", [])) + row["aliases"].add(name) + row["chunk_ids"].add(extraction.chunk_id) + for relationship in payload["relationships"]: + pending_relationships.append((extraction.chunk_id, relationship)) + for chunk_id, relationship in pending_relationships: + source_key = canonical_key(relationship["source"]) + target_key = canonical_key(relationship["target"]) + if source_key not in entity_rows or target_key not in entity_rows: + continue + relation_type = relationship["relationship_type"] + identity = (source_key, target_key, relation_type) + row = relation_rows.setdefault( + identity, {"description": "", "chunk_ids": set()} + ) + row["description"] = merge_text( + row["description"], relationship.get("description", "") + ) + row["chunk_ids"].add(chunk_id) + for key, row in entity_rows.items(): + session.add( + ResolvedEntity( + corpus=context.settings.corpus_name, + canonical_key=key, + canonical_name=row["canonical_name"], + entity_type=row["entity_type"], + description=row["description"], + aliases_json=sorted(row["aliases"], key=str.casefold), + source_chunk_ids_json=sorted(row["chunk_ids"]), + ) + ) + entity_count += 1 + for (source_key, target_key, relation_type), row in relation_rows.items(): + session.add( + ResolvedRelationship( + corpus=context.settings.corpus_name, + source_key=source_key, + target_key=target_key, + relationship_type=relation_type, + description=row["description"], + source_chunk_ids_json=sorted(row["chunk_ids"]), + ) + ) + relationship_count += 1 + return {"entities": entity_count, "relationships": relationship_count} + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + del args + return run(context) + + +if __name__ == "__main__": + stage_main(5, run_from_args) diff --git a/src/06_build_knowledge_graph.py b/src/06_build_knowledge_graph.py new file mode 100644 index 0000000..a760cbc --- /dev/null +++ b/src/06_build_knowledge_graph.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import argparse + +from sqlalchemy import delete, select + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import ( + Edge, + Node, + ResolvedEntity, + ResolvedRelationship, +) +from graphrag_platform.runtime import StageContext + + +def run(context: StageContext) -> dict[str, int]: + context.initialize() + with context.storage.session() as session: + entities = session.scalars( + select(ResolvedEntity).where( + ResolvedEntity.corpus == context.settings.corpus_name + ) + ).all() + relationships = session.scalars( + select(ResolvedRelationship).where( + ResolvedRelationship.corpus == context.settings.corpus_name + ) + ).all() + if not entities: + raise ValueError("stage 6 requires reconciled entities") + session.execute( + delete(Edge).where(Edge.corpus == context.settings.corpus_name) + ) + session.execute( + delete(Node).where(Node.corpus == context.settings.corpus_name) + ) + node_ids: dict[str, int] = {} + for entity in entities: + node = Node( + corpus=entity.corpus, + canonical_key=entity.canonical_key, + canonical_name=entity.canonical_name, + entity_type=entity.entity_type, + description=entity.description, + aliases_json=entity.aliases_json, + source_chunk_ids_json=entity.source_chunk_ids_json, + ) + session.add(node) + session.flush() + node_ids[entity.canonical_key] = node.id + edge_count = 0 + for relationship in relationships: + source_id = node_ids.get(relationship.source_key) + target_id = node_ids.get(relationship.target_key) + if source_id is None or target_id is None: + continue + session.add( + Edge( + corpus=relationship.corpus, + source_node_id=source_id, + target_node_id=target_id, + relationship_type=relationship.relationship_type, + description=relationship.description, + source_chunk_ids_json=relationship.source_chunk_ids_json, + ) + ) + edge_count += 1 + return {"nodes": len(node_ids), "edges": edge_count} + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + del args + return run(context) + + +if __name__ == "__main__": + stage_main(6, run_from_args) diff --git a/src/07_build_communities.py b/src/07_build_communities.py new file mode 100644 index 0000000..89ff15f --- /dev/null +++ b/src/07_build_communities.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import argparse +import json + +import networkx as nx +from sqlalchemy import delete, select + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import Community, Edge, Node +from graphrag_platform.runtime import StageContext + + +def run(context: StageContext) -> dict[str, int]: + context.initialize() + with context.storage.session() as session: + nodes = session.scalars( + select(Node).where(Node.corpus == context.settings.corpus_name) + ).all() + edges = session.scalars( + select(Edge).where(Edge.corpus == context.settings.corpus_name) + ).all() + if not nodes: + raise ValueError("stage 7 requires graph nodes") + graph = nx.Graph() + graph.add_nodes_from(node.id for node in nodes) + graph.add_edges_from( + (edge.source_node_id, edge.target_node_id) for edge in edges + ) + if graph.number_of_edges(): + groups = list( + nx.community.greedy_modularity_communities( + graph, resolution=context.settings.community_resolution + ) + ) + else: + groups = [{node.id} for node in nodes] + by_id = {node.id: node for node in nodes} + community_inputs = [ + ( + index, + sorted(group), + [ + { + "name": by_id[node_id].canonical_name, + "type": by_id[node_id].entity_type, + "description": by_id[node_id].description, + } + for node_id in sorted(group) + ], + ) + for index, group in enumerate(groups, start=1) + ] + + summaries: list[tuple[int, list[int], str, str]] = [] + for index, node_ids, evidence in community_inputs: + summary = context.llm.complete_json( + ( + "Summarize a knowledge-graph community using only the supplied " + 'evidence. Return JSON with non-empty "title" and "summary" strings.' + ), + json.dumps(evidence, ensure_ascii=False), + ) + title = str(summary.get("title", "")).strip() + description = str(summary.get("summary", "")).strip() + if not title or not description: + raise ValueError("community summary requires title and summary") + summaries.append((index, node_ids, title, description)) + + with context.storage.session() as session: + session.execute( + delete(Community).where( + Community.corpus == context.settings.corpus_name + ) + ) + for index, node_ids, title, description in summaries: + session.add( + Community( + corpus=context.settings.corpus_name, + community_key=f"community-{index}", + title=title, + summary=description, + node_ids_json=node_ids, + ) + ) + return {"communities": len(groups)} + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + del args + return run(context) + + +if __name__ == "__main__": + stage_main(7, run_from_args) diff --git a/src/08_build_retrieval_index.py b/src/08_build_retrieval_index.py new file mode 100644 index 0000000..bdf5edb --- /dev/null +++ b/src/08_build_retrieval_index.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import argparse +from typing import Any + +from sqlalchemy import delete, select + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import ( + Chunk, + Community, + Document, + Edge, + Node, + SearchItem, +) +from graphrag_platform.runtime import StageContext + + +def run(context: StageContext) -> dict[str, int]: + context.initialize() + with context.storage.session() as session: + chunks = session.scalars( + select(Chunk).where(Chunk.corpus == context.settings.corpus_name) + ).all() + nodes = session.scalars( + select(Node).where(Node.corpus == context.settings.corpus_name) + ).all() + edges = session.scalars( + select(Edge).where(Edge.corpus == context.settings.corpus_name) + ).all() + communities = session.scalars( + select(Community).where( + Community.corpus == context.settings.corpus_name + ) + ).all() + if not nodes or not chunks: + raise ValueError("stage 8 requires graph nodes and source chunks") + documents = { + document.id: document + for document in session.scalars( + select(Document).where( + Document.corpus == context.settings.corpus_name + ) + ).all() + } + node_names = {node.id: node.canonical_name for node in nodes} + node_source_chunks = { + node.id: set(node.source_chunk_ids_json) for node in nodes + } + candidates: list[dict[str, Any]] = [] + for chunk in chunks: + document = documents[chunk.document_id] + candidates.append( + { + "item_type": "chunk", + "item_key": str(chunk.id), + "text": chunk.text, + "source_url": document.url, + "metadata": { + "document_id": document.id, + "title": document.title, + "chunk_ordinal": chunk.ordinal, + }, + } + ) + for node in nodes: + candidates.append( + { + "item_type": "node", + "item_key": str(node.id), + "text": ( + f"{node.canonical_name} ({node.entity_type}): " + f"{node.description}" + ), + "source_url": None, + "metadata": {"source_chunk_ids": node.source_chunk_ids_json}, + } + ) + for edge in edges: + candidates.append( + { + "item_type": "edge", + "item_key": str(edge.id), + "text": ( + f"{node_names[edge.source_node_id]} " + f"{edge.relationship_type} " + f"{node_names[edge.target_node_id]}: {edge.description}" + ), + "source_url": None, + "metadata": { + "source_node_id": edge.source_node_id, + "target_node_id": edge.target_node_id, + "relationship_type": edge.relationship_type, + "source_chunk_ids": edge.source_chunk_ids_json, + }, + } + ) + for community in communities: + community_chunk_ids: set[int] = set() + for node_id in community.node_ids_json: + community_chunk_ids.update(node_source_chunks.get(node_id, set())) + candidates.append( + { + "item_type": "community", + "item_key": str(community.id), + "text": f"{community.title}: {community.summary}", + "source_url": None, + "metadata": { + "node_ids": community.node_ids_json, + "source_chunk_ids": sorted(community_chunk_ids), + }, + } + ) + embeddings: list[list[float]] = [] + for offset in range(0, len(candidates), 64): + embeddings.extend( + context.llm.embed( + [row["text"] for row in candidates[offset : offset + 64]] + ) + ) + if len(embeddings) != len(candidates): + raise ValueError("embedding provider returned an unexpected row count") + dimensions = context.settings.llm_embedding_dimensions + invalid_dimensions = [ + len(embedding) for embedding in embeddings if len(embedding) != dimensions + ] + if invalid_dimensions: + raise ValueError( + "embedding provider returned vectors with unexpected dimensions: " + f"expected {dimensions}, received {invalid_dimensions[0]}" + ) + with context.storage.session() as session: + session.execute( + delete(SearchItem).where( + SearchItem.corpus == context.settings.corpus_name + ) + ) + for candidate, embedding in zip(candidates, embeddings, strict=True): + session.add( + SearchItem( + corpus=context.settings.corpus_name, + item_type=candidate["item_type"], + item_key=candidate["item_key"], + text=candidate["text"], + source_url=candidate["source_url"], + metadata_json=candidate["metadata"], + embedding=embedding, + embedding_model=context.llm.embedding_model, + ) + ) + context.storage.ensure_vector_index(context.settings.llm_embedding_dimensions) + return {"indexed_items": len(candidates)} + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + del args + return run(context) + + +if __name__ == "__main__": + stage_main(8, run_from_args) diff --git a/src/09_query_graphrag.py b/src/09_query_graphrag.py new file mode 100644 index 0000000..84c8661 --- /dev/null +++ b/src/09_query_graphrag.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +import argparse +import math +from typing import Any + +from pgvector.sqlalchemy import Vector +from sqlalchemy import func, or_, select, text +from sqlalchemy.orm import Session + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import Chunk, Document, Edge, Node, SearchItem +from graphrag_platform.runtime import StageContext + + +def cosine_similarity(left: list[float], right: list[float]) -> float: + if len(left) != len(right) or not left: + return 0.0 + denominator = math.sqrt(sum(x * x for x in left)) * math.sqrt( + sum(x * x for x in right) + ) + if denominator == 0: + return 0.0 + return sum(x * y for x, y in zip(left, right, strict=True)) / denominator + + +def sources_for_chunks( + session: Session, chunk_ids: set[int] +) -> list[dict[str, Any]]: + if not chunk_ids: + return [] + rows = session.execute( + select(Chunk, Document) + .join(Document, Chunk.document_id == Document.id) + .where(Chunk.id.in_(chunk_ids)) + .order_by(Document.url, Chunk.ordinal) + ).all() + return [ + { + "citation": f"S{index}", + "url": document.url, + "title": document.title, + "chunk_id": chunk.id, + "text": chunk.text, + } + for index, (chunk, document) in enumerate(rows, start=1) + ] + + +def rank_vector_context( + context: StageContext, + session: Session, + query_embedding: list[float], + limit: int, +) -> list[tuple[float, SearchItem]]: + if context.storage.supports_vector_search: + dimensions = context.settings.llm_embedding_dimensions + session.execute( + text( + "SET LOCAL hnsw.ef_search = " + f"{context.settings.retrieval_hnsw_ef_search}" + ) + ) + session.execute(text("SET LOCAL hnsw.iterative_scan = strict_order")) + distance = ( + SearchItem.embedding.cast(Vector(dimensions)) + .cosine_distance(query_embedding) + .label("distance") + ) + rows = session.execute( + select(SearchItem, distance) + .where( + SearchItem.corpus == context.settings.corpus_name, + SearchItem.embedding_model == context.llm.embedding_model, + func.vector_dims(SearchItem.embedding) == dimensions, + ) + .order_by(distance) + .limit(limit) + ).all() + return [(1.0 - float(distance_value), item) for item, distance_value in rows] + + search_items = session.scalars( + select(SearchItem).where( + SearchItem.corpus == context.settings.corpus_name, + SearchItem.embedding_model == context.llm.embedding_model, + ) + ).all() + return sorted( + ( + (cosine_similarity(query_embedding, list(item.embedding)), item) + for item in search_items + ), + key=lambda pair: pair[0], + reverse=True, + )[:limit] + + +def expand_graph( + context: StageContext, + session: Session, + ranked: list[tuple[float, SearchItem]], +) -> tuple[list[dict[str, Any]], set[int]]: + seed_node_ids: set[int] = set() + source_chunk_ids: set[int] = set() + for _, item in ranked: + source_chunk_ids.update(item.metadata_json.get("source_chunk_ids", [])) + if item.item_type == "node": + seed_node_ids.add(int(item.item_key)) + elif item.item_type == "edge": + seed_node_ids.update( + { + int(item.metadata_json["source_node_id"]), + int(item.metadata_json["target_node_id"]), + } + ) + elif item.item_type == "community": + seed_node_ids.update(item.metadata_json.get("node_ids", [])) + + if not seed_node_ids or context.settings.retrieval_graph_hops == 0: + return [], source_chunk_ids + + visited_nodes = set(seed_node_ids) + visited_edges: dict[int, Edge] = {} + frontier = set(seed_node_ids) + for _ in range(context.settings.retrieval_graph_hops): + if not frontier or len(visited_edges) >= context.settings.retrieval_graph_limit: + break + remaining = context.settings.retrieval_graph_limit - len(visited_edges) + edges = session.scalars( + select(Edge) + .where( + Edge.corpus == context.settings.corpus_name, + or_( + Edge.source_node_id.in_(frontier), + Edge.target_node_id.in_(frontier), + ), + ) + .order_by(Edge.weight.desc(), Edge.id) + .limit(remaining) + ).all() + next_frontier: set[int] = set() + for edge in edges: + visited_edges[edge.id] = edge + source_chunk_ids.update(edge.source_chunk_ids_json) + next_frontier.update({edge.source_node_id, edge.target_node_id}) + next_frontier -= visited_nodes + visited_nodes.update(next_frontier) + frontier = next_frontier + + nodes = { + node.id: node + for node in session.scalars( + select(Node).where( + Node.corpus == context.settings.corpus_name, + Node.id.in_(visited_nodes), + ) + ).all() + } + for node in nodes.values(): + source_chunk_ids.update(node.source_chunk_ids_json) + graph_context = [ + { + "edge_id": edge.id, + "source_node_id": edge.source_node_id, + "source": nodes[edge.source_node_id].canonical_name, + "relationship": edge.relationship_type, + "target_node_id": edge.target_node_id, + "target": nodes[edge.target_node_id].canonical_name, + "description": edge.description, + "weight": edge.weight, + "source_chunk_ids": edge.source_chunk_ids_json, + } + for edge in visited_edges.values() + if edge.source_node_id in nodes and edge.target_node_id in nodes + ] + return graph_context, source_chunk_ids + + +def run( + context: StageContext, + *, + question: str, + top_k: int | None = None, +) -> dict[str, Any]: + context.initialize() + if not question.strip(): + raise ValueError("question must not be empty") + query_embedding = context.llm.embed([question])[0] + expected_dimensions = context.settings.llm_embedding_dimensions + if len(query_embedding) != expected_dimensions: + raise ValueError( + "embedding provider returned a query vector with unexpected dimensions: " + f"expected {expected_dimensions}, received {len(query_embedding)}" + ) + with context.storage.session() as session: + limit = top_k or context.settings.retrieval_top_k + ranked = rank_vector_context( + context, + session, + query_embedding, + limit, + ) + if not ranked: + raise ValueError("stage 9 requires a retrieval index") + graph_context, chunk_ids = expand_graph(context, session, ranked) + for _, item in ranked: + if item.item_type == "chunk": + chunk_ids.add(int(item.item_key)) + chunk_ids.update(item.metadata_json.get("source_chunk_ids", [])) + sources = sources_for_chunks(session, chunk_ids) + return { + "question": question, + "context": [ + { + "score": round(score, 6), + "type": item.item_type, + "key": item.item_key, + "text": item.text, + "source_url": item.source_url, + "metadata": item.metadata_json, + } + for score, item in ranked + ], + "graph_context": graph_context, + "sources": sources, + } + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, Any]: + if not args.question: + raise ValueError("--question is required for stage 9") + return run(context, question=args.question, top_k=args.top_k) + + +if __name__ == "__main__": + stage_main(9, run_from_args) diff --git a/src/10_generate_answer.py b/src/10_generate_answer.py new file mode 100644 index 0000000..dd6c26c --- /dev/null +++ b/src/10_generate_answer.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import argparse +import json +from typing import Any + +from graphrag_platform.cli import stage_main +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import load_stage + + +def run( + context: StageContext, + *, + question: str, + top_k: int | None = None, +) -> dict[str, Any]: + retrieval = load_stage(9).run( + context, + question=question, + top_k=top_k, + ) + answer = context.llm.complete_text( + ( + "Answer only from the supplied GraphRAG context. Cite supporting sources " + "using [S1], [S2], and so on. If the context does not support an answer, " + "state that clearly. Do not invent citations." + ), + json.dumps(retrieval, ensure_ascii=False), + ) + return { + "question": question, + "answer": answer, + "sources": retrieval["sources"], + } + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, Any]: + if not args.question: + raise ValueError("--question is required for stage 10") + return run(context, question=args.question, top_k=args.top_k) + + +if __name__ == "__main__": + stage_main(10, run_from_args) diff --git a/src/graphrag_platform/__init__.py b/src/graphrag_platform/__init__.py new file mode 100644 index 0000000..b5ea6eb --- /dev/null +++ b/src/graphrag_platform/__init__.py @@ -0,0 +1,6 @@ +"""KnowledgeGraph GraphRAG platform.""" + +from graphrag_platform.config import Settings +from graphrag_platform.pipeline import KnowledgeGraphPipeline + +__all__ = ["KnowledgeGraphPipeline", "Settings"] \ No newline at end of file diff --git a/src/graphrag_platform/api.py b/src/graphrag_platform/api.py new file mode 100644 index 0000000..e0de338 --- /dev/null +++ b/src/graphrag_platform/api.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +from typing import Annotated, Any, Callable + +import jwt +from fastapi import Depends, FastAPI, HTTPException, Request, status +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from pydantic import BaseModel, Field +from sqlalchemy import select, text + +from graphrag_platform.auth import Permission, Principal, decode_internal_token +from graphrag_platform.config import Settings +from graphrag_platform.llm import LLMClient +from graphrag_platform.models import PipelineRun, StageRun +from graphrag_platform.observability import configure_logging +from graphrag_platform.orchestrator import requeue_failed_run, submit_run +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import load_stage +from graphrag_platform.storage import Storage + +bearer = HTTPBearer(auto_error=False) + + +class JobRequest(BaseModel): + corpus: str = Field(min_length=1, max_length=200) + from_stage: int = Field(default=1, ge=1, le=8) + through_stage: int = Field(default=8, ge=1, le=8) + input_jsonl: str | None = None + model_file: str | None = None + reset: bool = False + + +class QueryRequest(BaseModel): + corpus: str = Field(min_length=1, max_length=200) + question: str = Field(min_length=1) + top_k: int | None = Field(default=None, ge=1) + + +def _principal( + request: Request, + credentials: Annotated[ + HTTPAuthorizationCredentials | None, Depends(bearer) + ], +) -> Principal: + if credentials is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Bearer service token is required", + ) + try: + return decode_internal_token(credentials.credentials, request.app.state.settings) + except jwt.PyJWTError as error: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid service token", + ) from error + + +def require_permissions( + required: Permission, +) -> Callable[[Principal], Principal]: + def dependency(principal: Annotated[Principal, Depends(_principal)]) -> Principal: + if not principal.has(required): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"KnowledgeGraph permission mask {int(required)} is required", + ) + return principal + + return dependency + + +def create_app( + settings: Settings | None = None, + *, + llm: LLMClient | None = None, +) -> FastAPI: + active_settings = settings or Settings() + configure_logging(active_settings.log_level) + app = FastAPI( + title="KnowledgeGraph Internal API", + version="1.0.0", + docs_url=None, + redoc_url=None, + openapi_url="/internal/knowledge-graph/openapi.json", + ) + app.state.settings = active_settings + app.state.storage = Storage(active_settings.database_url) + app.state.llm = llm + + def context(corpus: str) -> StageContext: + corpus_settings = active_settings.model_copy(update={"corpus_name": corpus}) + return StageContext.create( + corpus_settings, + storage=app.state.storage, + llm=app.state.llm, + ) + + @app.get("/internal/knowledge-graph/health") + def health() -> dict[str, str]: + return {"status": "healthy"} + + @app.get("/internal/knowledge-graph/ready") + def ready() -> dict[str, str]: + try: + with app.state.storage.engine.connect() as connection: + connection.execute(text("SELECT 1")) + if connection.dialect.name == "postgresql": + extension = connection.scalar( + text( + "SELECT extname FROM pg_extension " + "WHERE extname = 'vector'" + ) + ) + if extension != "vector": + raise RuntimeError("PgVector extension is not installed") + except Exception as error: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Service dependencies are not ready", + ) from error + return {"status": "ready"} + + @app.post( + "/internal/knowledge-graph/jobs", + status_code=status.HTTP_202_ACCEPTED, + ) + def create_job( + body: JobRequest, + principal: Annotated[ + Principal, Depends(require_permissions(Permission.EXECUTE)) + ], + ) -> dict[str, str]: + if body.from_stage == 1 and not principal.has(Permission.ADD): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="KnowledgeGraph Add permission is required for stage 1", + ) + if body.from_stage > body.through_stage: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="from_stage must not exceed through_stage", + ) + run_context = context(body.corpus) + run_context.initialize() + run_id = submit_run( + run_context, + from_stage=body.from_stage, + through_stage=body.through_stage, + payload={ + "input_jsonl": body.input_jsonl, + "model_file": body.model_file, + "reset": body.reset, + }, + requested_by=principal.subject, + correlation_id=principal.correlation_id, + permission_mask=principal.permission_mask, + ) + return {"job_id": run_id, "status": "queued"} + + @app.get("/internal/knowledge-graph/jobs/{job_id}") + def get_job( + job_id: str, + principal: Annotated[ + Principal, Depends(require_permissions(Permission.READ)) + ], + ) -> dict[str, Any]: + del principal + with app.state.storage.session() as session: + run = session.get(PipelineRun, job_id) + if run is None: + raise HTTPException(status_code=404, detail="Job not found") + stages = session.scalars( + select(StageRun) + .where(StageRun.pipeline_run_id == job_id) + .order_by(StageRun.stage_number, StageRun.attempt) + ).all() + return { + "job_id": run.id, + "corpus": run.corpus, + "status": run.status, + "attempt": run.attempt, + "from_stage": run.from_stage, + "through_stage": run.through_stage, + "error": run.error_message, + "output": run.output_json, + "stages": [ + { + "stage": stage_run.stage_number, + "name": stage_run.stage_name, + "status": stage_run.status, + "attempt": stage_run.attempt, + "output": stage_run.output_json, + "error": stage_run.error_message, + } + for stage_run in stages + ], + } + + @app.post( + "/internal/knowledge-graph/jobs/{job_id}/retry", + status_code=status.HTTP_202_ACCEPTED, + ) + def retry_job( + job_id: str, + principal: Annotated[ + Principal, Depends(require_permissions(Permission.EXECUTE)) + ], + ) -> dict[str, str]: + del principal + requeue_failed_run(context(active_settings.corpus_name), job_id) + return {"job_id": job_id, "status": "queued"} + + @app.post("/internal/knowledge-graph/query") + def query( + body: QueryRequest, + principal: Annotated[ + Principal, + Depends(require_permissions(Permission.READ | Permission.EXECUTE)), + ], + ) -> dict[str, Any]: + del principal + return load_stage(9).run( + context(body.corpus), + question=body.question, + top_k=body.top_k, + ) + + @app.post("/internal/knowledge-graph/answer") + def answer( + body: QueryRequest, + principal: Annotated[ + Principal, + Depends(require_permissions(Permission.READ | Permission.EXECUTE)), + ], + ) -> dict[str, Any]: + del principal + return load_stage(10).run( + context(body.corpus), + question=body.question, + top_k=body.top_k, + ) + + return app + + +app = create_app() diff --git a/src/graphrag_platform/auth.py b/src/graphrag_platform/auth.py new file mode 100644 index 0000000..db58a83 --- /dev/null +++ b/src/graphrag_platform/auth.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import IntFlag +from typing import Any + +import jwt + +from graphrag_platform.config import Settings + + +class Permission(IntFlag): + READ = 1 + ADD = 2 + EDIT = 4 + DELETE = 8 + EXECUTE = 16 + + +@dataclass(frozen=True) +class Principal: + subject: str + permission_mask: int + correlation_id: str + claims: dict[str, Any] + + def has(self, required: Permission) -> bool: + return (self.permission_mask & int(required)) == int(required) + + +def decode_internal_token(token: str, settings: Settings) -> Principal: + claims = jwt.decode( + token, + settings.internal_jwt_secret, + algorithms=["HS256"], + issuer=settings.internal_jwt_issuer, + audience=settings.internal_jwt_audience, + options={"require": ["exp", "iat", "iss", "aud", "sub", "features"]}, + ) + features = claims.get("features") + if not isinstance(features, dict): + raise jwt.InvalidTokenError("features claim must be an object") + raw_mask = features.get(settings.internal_feature_name) + if not isinstance(raw_mask, int): + raise jwt.InvalidTokenError( + f"missing integer feature claim: {settings.internal_feature_name}" + ) + correlation_id = str(claims.get("correlation_id") or claims.get("jti") or "") + if not correlation_id: + raise jwt.InvalidTokenError("correlation_id or jti is required") + return Principal( + subject=str(claims["sub"]), + permission_mask=raw_mask, + correlation_id=correlation_id, + claims=claims, + ) diff --git a/src/graphrag_platform/cli.py b/src/graphrag_platform/cli.py new file mode 100644 index 0000000..331e9ae --- /dev/null +++ b/src/graphrag_platform/cli.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import argparse +import json +import sys +from uuid import uuid4 +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from graphrag_platform.config import Settings +from graphrag_platform.observability import configure_logging +from graphrag_platform.orchestrator import execute_run, submit_run +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import STAGE_MODULES, load_stage + +StageRunner = Callable[[StageContext, argparse.Namespace], dict[str, Any]] + + +def create_parser(fixed_stage: int | None) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run a KnowledgeGraph GraphRAG pipeline stage." + ) + if fixed_stage is None: + parser.add_argument("stage", type=int, choices=range(1, 11)) + parser.add_argument("--env-file", type=Path, default=Path(".env")) + parser.add_argument("--database-url", help="Override DATABASE_URL.") + parser.add_argument("--corpus", help="Override CORPUS_NAME.") + parser.add_argument("--input-jsonl", type=Path) + parser.add_argument("--model-file", type=Path) + parser.add_argument("--question") + parser.add_argument("--top-k", type=int) + parser.add_argument("--reset", action="store_true") + return parser + + +def create_run_parser() -> argparse.ArgumentParser: + parser = create_parser(1) + parser.description = "Run a durable KnowledgeGraph build pipeline." + parser.add_argument("--from-stage", type=int, choices=range(1, 9), default=1) + parser.add_argument("--through-stage", type=int, choices=range(1, 9), default=8) + parser.add_argument("--requested-by", default="local-cli") + parser.add_argument("--correlation-id", default=None) + return parser + + +def create_context( + args: argparse.Namespace, stage: int, *, apply_reset: bool = True +) -> StageContext: + overrides: dict[str, Any] = {} + if args.database_url: + overrides["database_url"] = args.database_url + if args.corpus: + overrides["corpus_name"] = args.corpus + settings = Settings(_env_file=args.env_file, **overrides) + context = StageContext.create(settings) + context.initialize() + if args.reset and apply_reset: + context.storage.clear_from_stage(settings.corpus_name, stage) + return context + + +def run(stage: int, args: argparse.Namespace) -> dict[str, Any]: + context = create_context(args, stage) + module = load_stage(stage) + return module.run_from_args(context, args) + + +def stage_main(stage: int, runner: StageRunner) -> None: + args = create_parser(stage).parse_args() + context = create_context(args, stage) + configure_logging(context.settings.log_level) + result = runner(context, args) + print(json.dumps(result, indent=2, ensure_ascii=False)) + + +def main_all() -> None: + if len(sys.argv) > 1 and sys.argv[1] == "run": + args = create_run_parser().parse_args(sys.argv[2:]) + context = create_context(args, args.from_stage, apply_reset=False) + configure_logging(context.settings.log_level) + run_id = submit_run( + context, + from_stage=args.from_stage, + through_stage=args.through_stage, + payload={ + "input_jsonl": str(args.input_jsonl) if args.input_jsonl else None, + "model_file": str(args.model_file) if args.model_file else None, + "reset": args.reset, + }, + requested_by=args.requested_by, + correlation_id=args.correlation_id or str(uuid4()), + permission_mask=31, + ) + outputs = execute_run(context, run_id) + print(json.dumps({"run_id": run_id, "outputs": outputs}, indent=2)) + return + args = create_parser(None).parse_args() + configure_logging(Settings(_env_file=args.env_file).log_level) + result = run(args.stage, args) + print( + json.dumps( + { + "stage": args.stage, + "name": STAGE_MODULES[args.stage].removeprefix(f"{args.stage:02d}_"), + **result, + }, + indent=2, + ensure_ascii=False, + ) + ) diff --git a/src/graphrag_platform/config.py b/src/graphrag_platform/config.py new file mode 100644 index 0000000..2930fe4 --- /dev/null +++ b/src/graphrag_platform/config.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Annotated, Any + +from pydantic import Field, field_validator +from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict + + +class Settings(BaseSettings): + """Runtime configuration loaded from environment variables or a .env file.""" + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + case_sensitive=False, + ) + + database_url: str = ( + "postgresql+psycopg://knowledgegraph_app:change-me" + "@localhost:5432/knowledgegraph" + ) + source_database_url: str | None = None + source_table_name: str = "external_documents" + analytics_database_url: str | None = None + auto_create_schema: bool = False + + corpus_name: str = "default-corpus" + + entity_types: Annotated[list[str], NoDecode] = Field( + default_factory=lambda: [ + "Person", + "Organization", + "Product", + "Concept", + "Event", + "Location", + ] + ) + relationship_types: Annotated[list[str], NoDecode] = Field( + default_factory=lambda: [ + "AFFILIATED_WITH", + "CREATED", + "DEPENDS_ON", + "MENTIONS", + "PART_OF", + "RELATED_TO", + ] + ) + node_template: dict[str, Any] = Field( + default_factory=lambda: { + "canonical_name": "string", + "entity_type": "one configured entity type", + "description": "source-grounded string", + "aliases": ["string"], + } + ) + edge_template: dict[str, Any] = Field( + default_factory=lambda: { + "source": "entity name", + "target": "entity name", + "relationship_type": "one configured relationship type", + "description": "source-grounded string", + } + ) + extraction_instructions: str = ( + "Extract only entities and relationships explicitly supported by the supplied text." + ) + + llm_base_url: str = "https://api.openai.com/v1" + llm_api_key: str | None = None + llm_chat_model: str = "gpt-4.1-mini" + llm_embedding_model: str = "text-embedding-3-small" + llm_embedding_dimensions: int = Field(default=1536, ge=1) + llm_timeout_seconds: float = Field(default=60, gt=0) + llm_max_retries: int = Field(default=4, ge=0, le=10) + llm_retry_initial_seconds: float = Field(default=1, gt=0) + llm_retry_max_seconds: float = Field(default=30, gt=0) + + chunk_size: int = Field(default=1200, ge=200) + chunk_overlap: int = Field(default=150, ge=0) + community_resolution: float = Field(default=1.0, gt=0) + retrieval_top_k: int = Field(default=8, ge=1) + retrieval_graph_hops: int = Field(default=2, ge=0, le=5) + retrieval_graph_limit: int = Field(default=50, ge=1) + retrieval_hnsw_ef_search: int = Field(default=100, ge=1) + log_level: str = "INFO" + internal_jwt_secret: str = "replace-in-dev-test" + internal_jwt_issuer: str = "knowledgegraph-bff" + internal_jwt_audience: str = "knowledgegraph-service" + internal_feature_name: str = "KnowledgeGraph" + worker_poll_seconds: float = Field(default=2, gt=0) + worker_stale_run_seconds: int = Field(default=3600, ge=60) + artifact_dir: Path = Path("artifacts") + + @field_validator( + "entity_types", + "relationship_types", + mode="before", + ) + @classmethod + def parse_comma_separated_list(cls, value: object) -> object: + if isinstance(value, str): + return [item.strip() for item in value.split(",") if item.strip()] + return value + + @field_validator("node_template", "edge_template", mode="before") + @classmethod + def parse_json_object(cls, value: object) -> object: + if isinstance(value, str): + parsed = json.loads(value) + if not isinstance(parsed, dict): + raise ValueError("template must be a JSON object") + return parsed + return value + + def model_definition(self) -> dict[str, Any]: + return { + "entity_types": self.entity_types, + "relationship_types": self.relationship_types, + "node_template": self.node_template, + "edge_template": self.edge_template, + "extraction_instructions": self.extraction_instructions, + } \ No newline at end of file diff --git a/src/graphrag_platform/llm.py b/src/graphrag_platform/llm.py new file mode 100644 index 0000000..8fe00ce --- /dev/null +++ b/src/graphrag_platform/llm.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import json +import logging +import random +import time +from collections.abc import Sequence +from typing import Any, Callable, Protocol + +import httpx + +from graphrag_platform.config import Settings + +LOGGER = logging.getLogger(__name__) +_RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504} + + +class LLMClient(Protocol): + chat_model: str + embedding_model: str + + def complete_json(self, system: str, user: str) -> dict[str, Any]: ... + + def complete_text(self, system: str, user: str) -> str: ... + + def embed(self, texts: Sequence[str]) -> list[list[float]]: ... + + +class OpenAICompatibleClient: + def __init__( + self, + settings: Settings, + transport: httpx.BaseTransport | None = None, + sleeper: Callable[[float], None] = time.sleep, + jitter: Callable[[], float] = random.random, + ) -> None: + if not settings.llm_api_key: + raise ValueError("LLM_API_KEY is required for LLM-backed stages") + self.chat_model = settings.llm_chat_model + self.embedding_model = settings.llm_embedding_model + self._settings = settings + self._sleeper = sleeper + self._jitter = jitter + self._client = httpx.Client( + base_url=settings.llm_base_url.rstrip("/") + "/", + timeout=settings.llm_timeout_seconds, + headers={ + "Authorization": f"Bearer {settings.llm_api_key}", + "Content-Type": "application/json", + }, + transport=transport, + ) + + def complete_json(self, system: str, user: str) -> dict[str, Any]: + response = self._post( + "chat/completions", + json={ + "model": self.chat_model, + "response_format": {"type": "json_object"}, + "temperature": 0, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + }, + ) + content = response.json()["choices"][0]["message"]["content"] + parsed = json.loads(content) + if not isinstance(parsed, dict): + raise ValueError("LLM response must be a JSON object") + return parsed + + def complete_text(self, system: str, user: str) -> str: + response = self._post( + "chat/completions", + json={ + "model": self.chat_model, + "temperature": 0, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + }, + ) + return str(response.json()["choices"][0]["message"]["content"]).strip() + + def embed(self, texts: Sequence[str]) -> list[list[float]]: + if not texts: + return [] + response = self._post( + "embeddings", + json={"model": self.embedding_model, "input": list(texts)}, + ) + rows = sorted(response.json()["data"], key=lambda row: row["index"]) + return [[float(value) for value in row["embedding"]] for row in rows] + + def _post(self, path: str, *, json: dict[str, Any]) -> httpx.Response: + attempts = self._settings.llm_max_retries + 1 + for attempt in range(1, attempts + 1): + try: + response = self._client.post(path, json=json) + if response.status_code not in _RETRYABLE_STATUS_CODES: + response.raise_for_status() + return response + if attempt == attempts: + response.raise_for_status() + delay = self._retry_delay(response, attempt) + LOGGER.warning( + "Retrying model provider request", + extra={ + "event": "llm_retry", + "attempt": attempt, + "status_code": response.status_code, + "delay_seconds": delay, + "path": path, + }, + ) + self._sleeper(delay) + except (httpx.ConnectError, httpx.TimeoutException) as error: + if attempt == attempts: + raise + delay = self._retry_delay(None, attempt) + LOGGER.warning( + "Retrying model provider request after transport error", + extra={ + "event": "llm_retry", + "attempt": attempt, + "error_type": type(error).__name__, + "delay_seconds": delay, + "path": path, + }, + ) + self._sleeper(delay) + raise RuntimeError("model provider retry loop ended unexpectedly") + + def _retry_delay( + self, response: httpx.Response | None, attempt: int + ) -> float: + if response is not None: + retry_after = response.headers.get("Retry-After") + if retry_after: + try: + return min( + float(retry_after), + self._settings.llm_retry_max_seconds, + ) + except ValueError: + pass + exponential = self._settings.llm_retry_initial_seconds * (2 ** (attempt - 1)) + return min( + exponential + self._jitter(), + self._settings.llm_retry_max_seconds, + ) \ No newline at end of file diff --git a/src/graphrag_platform/models.py b/src/graphrag_platform/models.py new file mode 100644 index 0000000..ad219ad --- /dev/null +++ b/src/graphrag_platform/models.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any +from uuid import uuid4 + +from sqlalchemy import ( + DateTime, + Float, + ForeignKey, + Index, + Integer, + JSON, + String, + Text, + UniqueConstraint, +) +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column +from pgvector.sqlalchemy import Vector + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class Base(DeclarativeBase): + pass + + +class RawDocument(Base): + __tablename__ = "raw_documents" + __table_args__ = ( + UniqueConstraint("corpus", "external_id", name="uq_raw_corpus_external_id"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + corpus: Mapped[str] = mapped_column(String(200), index=True) + external_id: Mapped[str] = mapped_column(String(500)) + source_schema_version: Mapped[str] = mapped_column(String(20)) + url: Mapped[str] = mapped_column(Text) + content_type: Mapped[str | None] = mapped_column(String(200)) + status_code: Mapped[int | None] = mapped_column(Integer) + body: Mapped[str] = mapped_column(Text) + metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + collected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now) + + +class Document(Base): + __tablename__ = "documents" + __table_args__ = ( + UniqueConstraint("corpus", "url", name="uq_document_corpus_url"), + UniqueConstraint("raw_document_id", name="uq_document_raw_document"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + raw_document_id: Mapped[int | None] = mapped_column( + ForeignKey("raw_documents.id", ondelete="SET NULL") + ) + corpus: Mapped[str] = mapped_column(String(200), index=True) + url: Mapped[str] = mapped_column(Text) + title: Mapped[str | None] = mapped_column(Text) + author: Mapped[str | None] = mapped_column(Text) + published_at: Mapped[str | None] = mapped_column(String(100)) + text: Mapped[str] = mapped_column(Text) + content_hash: Mapped[str] = mapped_column(String(64), index=True) + metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + is_duplicate: Mapped[bool] = mapped_column(default=False) + canonical_document_id: Mapped[int | None] = mapped_column( + ForeignKey("documents.id", ondelete="SET NULL") + ) + + +class Chunk(Base): + __tablename__ = "chunks" + __table_args__ = (UniqueConstraint("document_id", "ordinal", name="uq_chunk_ordinal"),) + + id: Mapped[int] = mapped_column(primary_key=True) + document_id: Mapped[int] = mapped_column( + ForeignKey("documents.id", ondelete="CASCADE"), index=True + ) + corpus: Mapped[str] = mapped_column(String(200), index=True) + ordinal: Mapped[int] = mapped_column(Integer) + text: Mapped[str] = mapped_column(Text) + content_hash: Mapped[str] = mapped_column(String(64), index=True) + + +class KnowledgeModel(Base): + __tablename__ = "knowledge_models" + + corpus: Mapped[str] = mapped_column(String(200), primary_key=True) + definition_json: Mapped[dict[str, Any]] = mapped_column(JSON) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utc_now, onupdate=utc_now + ) + + +class Extraction(Base): + __tablename__ = "extractions" + __table_args__ = (UniqueConstraint("chunk_id", name="uq_extraction_chunk"),) + + id: Mapped[int] = mapped_column(primary_key=True) + chunk_id: Mapped[int] = mapped_column( + ForeignKey("chunks.id", ondelete="CASCADE"), index=True + ) + corpus: Mapped[str] = mapped_column(String(200), index=True) + payload_json: Mapped[dict[str, Any]] = mapped_column(JSON) + model_name: Mapped[str] = mapped_column(String(200)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now) + + +class ResolvedEntity(Base): + __tablename__ = "resolved_entities" + __table_args__ = ( + UniqueConstraint("corpus", "canonical_key", name="uq_resolved_entity_key"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + corpus: Mapped[str] = mapped_column(String(200), index=True) + canonical_key: Mapped[str] = mapped_column(String(500)) + canonical_name: Mapped[str] = mapped_column(Text) + entity_type: Mapped[str] = mapped_column(String(100)) + description: Mapped[str] = mapped_column(Text, default="") + aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list) + source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list) + + +class ResolvedRelationship(Base): + __tablename__ = "resolved_relationships" + __table_args__ = ( + UniqueConstraint( + "corpus", + "source_key", + "target_key", + "relationship_type", + name="uq_resolved_relationship", + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + corpus: Mapped[str] = mapped_column(String(200), index=True) + source_key: Mapped[str] = mapped_column(String(500)) + target_key: Mapped[str] = mapped_column(String(500)) + relationship_type: Mapped[str] = mapped_column(String(100)) + description: Mapped[str] = mapped_column(Text, default="") + source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list) + + +class Node(Base): + __tablename__ = "nodes" + __table_args__ = ( + UniqueConstraint("corpus", "canonical_key", name="uq_node_canonical_key"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + corpus: Mapped[str] = mapped_column(String(200), index=True) + canonical_key: Mapped[str] = mapped_column(String(500)) + canonical_name: Mapped[str] = mapped_column(Text) + entity_type: Mapped[str] = mapped_column(String(100), index=True) + description: Mapped[str] = mapped_column(Text, default="") + aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list) + source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list) + + +class Edge(Base): + __tablename__ = "edges" + __table_args__ = ( + UniqueConstraint( + "corpus", + "source_node_id", + "target_node_id", + "relationship_type", + name="uq_edge_identity", + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + corpus: Mapped[str] = mapped_column(String(200), index=True) + source_node_id: Mapped[int] = mapped_column( + ForeignKey("nodes.id", ondelete="CASCADE"), index=True + ) + target_node_id: Mapped[int] = mapped_column( + ForeignKey("nodes.id", ondelete="CASCADE"), index=True + ) + relationship_type: Mapped[str] = mapped_column(String(100), index=True) + description: Mapped[str] = mapped_column(Text, default="") + weight: Mapped[float] = mapped_column(Float, default=1.0) + source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list) + + +class Community(Base): + __tablename__ = "communities" + __table_args__ = (UniqueConstraint("corpus", "community_key", name="uq_community_key"),) + + id: Mapped[int] = mapped_column(primary_key=True) + corpus: Mapped[str] = mapped_column(String(200), index=True) + community_key: Mapped[str] = mapped_column(String(100)) + title: Mapped[str] = mapped_column(Text) + summary: Mapped[str] = mapped_column(Text) + node_ids_json: Mapped[list[int]] = mapped_column(JSON) + + +class SearchItem(Base): + __tablename__ = "retrieval_items" + __table_args__ = ( + UniqueConstraint("corpus", "item_type", "item_key", name="uq_search_item"), + Index("ix_retrieval_items_corpus_type", "corpus", "item_type"), + Index( + "ix_retrieval_items_corpus_model", + "corpus", + "embedding_model", + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + corpus: Mapped[str] = mapped_column(String(200), index=True) + item_type: Mapped[str] = mapped_column(String(50)) + item_key: Mapped[str] = mapped_column(String(100)) + text: Mapped[str] = mapped_column(Text) + source_url: Mapped[str | None] = mapped_column(Text) + metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + embedding: Mapped[list[float]] = mapped_column( + Vector().with_variant(JSON(), "sqlite") + ) + embedding_model: Mapped[str] = mapped_column(String(200)) + + +class PipelineRun(Base): + __tablename__ = "pipeline_runs" + __table_args__ = ( + Index("ix_pipeline_runs_status_created", "status", "created_at"), + Index("ix_pipeline_runs_corpus_created", "corpus", "created_at"), + ) + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid4()) + ) + corpus: Mapped[str] = mapped_column(String(200), index=True) + status: Mapped[str] = mapped_column(String(30), index=True, default="queued") + from_stage: Mapped[int] = mapped_column(Integer) + through_stage: Mapped[int] = mapped_column(Integer) + payload_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + configuration_hash: Mapped[str] = mapped_column(String(64)) + requested_by: Mapped[str] = mapped_column(String(500)) + correlation_id: Mapped[str] = mapped_column(String(200), index=True) + feature_name: Mapped[str] = mapped_column(String(200), default="KnowledgeGraph") + permission_mask: Mapped[int] = mapped_column(Integer) + attempt: Mapped[int] = mapped_column(Integer, default=0) + output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + error_message: Mapped[str | None] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + +class StageRun(Base): + __tablename__ = "stage_runs" + __table_args__ = ( + UniqueConstraint( + "pipeline_run_id", + "stage_number", + "attempt", + name="uq_stage_run_attempt", + ), + Index("ix_stage_runs_pipeline_stage", "pipeline_run_id", "stage_number"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + pipeline_run_id: Mapped[str] = mapped_column( + ForeignKey("pipeline_runs.id", ondelete="CASCADE"), index=True + ) + stage_number: Mapped[int] = mapped_column(Integer) + stage_name: Mapped[str] = mapped_column(String(200)) + status: Mapped[str] = mapped_column(String(30), default="running") + attempt: Mapped[int] = mapped_column(Integer) + input_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + error_message: Mapped[str | None] = mapped_column(Text) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) \ No newline at end of file diff --git a/src/graphrag_platform/observability.py b/src/graphrag_platform/observability.py new file mode 100644 index 0000000..785169e --- /dev/null +++ b/src/graphrag_platform/observability.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone +from typing import Any + + +class JsonFormatter(logging.Formatter): + _reserved = set(logging.makeLogRecord({}).__dict__) + + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, Any] = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + } + for key, value in record.__dict__.items(): + if key not in self._reserved and key not in {"message", "asctime"}: + payload[key] = value + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + return json.dumps(payload, ensure_ascii=False, default=str) + + +def configure_logging(level: str = "INFO") -> None: + handler = logging.StreamHandler() + handler.setFormatter(JsonFormatter()) + root = logging.getLogger() + root.handlers.clear() + root.addHandler(handler) + root.setLevel(level.upper()) diff --git a/src/graphrag_platform/orchestrator.py b/src/graphrag_platform/orchestrator.py new file mode 100644 index 0000000..a3da801 --- /dev/null +++ b/src/graphrag_platform/orchestrator.py @@ -0,0 +1,330 @@ +from __future__ import annotations + +import hashlib +import json +import logging +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from sqlalchemy import select, text, update + +from graphrag_platform.models import PipelineRun, StageRun +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import STAGE_MODULES, load_stage + +LOGGER = logging.getLogger(__name__) +BUILD_STAGES = range(1, 9) + + +class RunDeferredError(RuntimeError): + pass + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def configuration_hash(context: StageContext) -> str: + payload = context.settings.model_dump( + mode="json", + exclude={"llm_api_key", "internal_jwt_secret"}, + ) + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def submit_run( + context: StageContext, + *, + from_stage: int, + through_stage: int, + payload: dict[str, Any], + requested_by: str, + correlation_id: str, + permission_mask: int, +) -> str: + if from_stage not in BUILD_STAGES or through_stage not in BUILD_STAGES: + raise ValueError("pipeline jobs support build stages 1 through 8") + if from_stage > through_stage: + raise ValueError("from_stage must not exceed through_stage") + run = PipelineRun( + corpus=context.settings.corpus_name, + status="queued", + from_stage=from_stage, + through_stage=through_stage, + payload_json=payload, + configuration_hash=configuration_hash(context), + requested_by=requested_by, + correlation_id=correlation_id, + feature_name=context.settings.internal_feature_name, + permission_mask=permission_mask, + ) + with context.storage.session() as session: + session.add(run) + session.flush() + run_id = run.id + LOGGER.info( + "Pipeline run queued", + extra={ + "event": "pipeline_queued", + "run_id": run_id, + "corpus": context.settings.corpus_name, + "from_stage": from_stage, + "through_stage": through_stage, + "correlation_id": correlation_id, + }, + ) + return run_id + + +def claim_next_run(context: StageContext) -> str | None: + with context.storage.session() as session: + statement = ( + select(PipelineRun) + .where(PipelineRun.status == "queued") + .order_by(PipelineRun.created_at) + .limit(1) + ) + if context.storage.engine.dialect.name == "postgresql": + statement = statement.with_for_update(skip_locked=True) + run = session.scalar(statement) + if run is None: + return None + run.status = "running" + run.attempt += 1 + run.started_at = run.started_at or utc_now() + run.heartbeat_at = utc_now() + session.flush() + return run.id + + +def execute_run(context: StageContext, run_id: str) -> dict[str, Any]: + with context.storage.session() as session: + run = session.get(PipelineRun, run_id) + if run is None: + raise ValueError(f"pipeline run does not exist: {run_id}") + if run.status == "queued": + run.status = "running" + run.attempt += 1 + run.started_at = run.started_at or utc_now() + if run.status != "running": + raise ValueError(f"pipeline run {run_id} is {run.status}, not running") + from_stage = run.from_stage + through_stage = run.through_stage + payload = dict(run.payload_json) + attempt = run.attempt + execution_context = context.for_corpus(run.corpus) + corpus = run.corpus + + lock_connection = None + if context.storage.engine.dialect.name == "postgresql": + lock_connection = context.storage.engine.connect() + acquired = lock_connection.scalar( + text("SELECT pg_try_advisory_lock(hashtext(:key))"), + {"key": f"knowledgegraph:{corpus}"}, + ) + if not acquired: + lock_connection.close() + with context.storage.session() as session: + run = session.get(PipelineRun, run_id) + if run is not None: + run.status = "queued" + run.error_message = "Deferred because the corpus is already running" + raise RunDeferredError(f"corpus already has an active run: {corpus}") + + outputs: dict[str, Any] = {} + try: + if payload.get("reset") and attempt == 1: + execution_context.storage.clear_from_stage(corpus, from_stage) + for stage_number in range(from_stage, through_stage + 1): + previous_output = _successful_stage_output( + context, run_id, stage_number + ) + if previous_output is not None: + outputs[str(stage_number)] = previous_output + continue + stage_name = STAGE_MODULES[stage_number] + stage_run_id = _start_stage_run( + context, + run_id=run_id, + stage_number=stage_number, + stage_name=stage_name, + attempt=attempt, + payload=payload, + ) + try: + result = _execute_stage(execution_context, stage_number, payload) + except Exception as error: + _finish_stage_run( + context, + stage_run_id, + status="failed", + error_message=str(error), + ) + raise + _finish_stage_run( + context, + stage_run_id, + status="succeeded", + output=result, + ) + outputs[str(stage_number)] = result + _checkpoint(context, run_id, outputs) + with context.storage.session() as session: + run = session.get(PipelineRun, run_id) + if run is None: + raise RuntimeError(f"pipeline run disappeared: {run_id}") + run.status = "succeeded" + run.output_json = outputs + run.finished_at = utc_now() + run.heartbeat_at = utc_now() + LOGGER.info( + "Pipeline run succeeded", + extra={"event": "pipeline_succeeded", "run_id": run_id}, + ) + return outputs + except Exception as error: + with context.storage.session() as session: + run = session.get(PipelineRun, run_id) + if run is not None: + run.status = "failed" + run.error_message = str(error) + run.finished_at = utc_now() + run.heartbeat_at = utc_now() + LOGGER.exception( + "Pipeline run failed", + extra={"event": "pipeline_failed", "run_id": run_id}, + ) + raise + finally: + if lock_connection is not None: + lock_connection.execute( + text("SELECT pg_advisory_unlock(hashtext(:key))"), + {"key": f"knowledgegraph:{corpus}"}, + ) + lock_connection.close() + + +def requeue_failed_run(context: StageContext, run_id: str) -> None: + with context.storage.session() as session: + run = session.get(PipelineRun, run_id) + if run is None: + raise ValueError(f"pipeline run does not exist: {run_id}") + if run.status != "failed": + raise ValueError("only failed pipeline runs can be requeued") + run.status = "queued" + run.error_message = None + run.finished_at = None + + +def recover_stale_runs(context: StageContext) -> int: + threshold = utc_now().timestamp() - context.settings.worker_stale_run_seconds + cutoff = datetime.fromtimestamp(threshold, tz=timezone.utc) + with context.storage.session() as session: + result = session.execute( + update(PipelineRun) + .where( + PipelineRun.status == "running", + PipelineRun.heartbeat_at < cutoff, + ) + .values( + status="queued", + error_message="Recovered after a stale worker lease", + finished_at=None, + ) + ) + recovered = int(result.rowcount or 0) + if recovered: + LOGGER.warning( + "Recovered stale pipeline runs", + extra={"event": "pipeline_stale_recovered", "count": recovered}, + ) + return recovered + + +def _execute_stage( + context: StageContext, stage_number: int, payload: dict[str, Any] +) -> dict[str, Any]: + module = load_stage(stage_number) + if stage_number == 1: + input_path = payload.get("input_jsonl") + return module.run( + context, + input_jsonl=Path(input_path) if input_path else None, + ) + if stage_number == 3: + model_path = payload.get("model_file") + return module.run( + context, + model_file=Path(model_path) if model_path else None, + ) + return module.run(context) + + +def _start_stage_run( + context: StageContext, + *, + run_id: str, + stage_number: int, + stage_name: str, + attempt: int, + payload: dict[str, Any], +) -> int: + with context.storage.session() as session: + stage_run = StageRun( + pipeline_run_id=run_id, + stage_number=stage_number, + stage_name=stage_name, + status="running", + attempt=attempt, + input_json=payload, + ) + session.add(stage_run) + session.flush() + return stage_run.id + + +def _finish_stage_run( + context: StageContext, + stage_run_id: int, + *, + status: str, + output: dict[str, Any] | None = None, + error_message: str | None = None, +) -> None: + with context.storage.session() as session: + stage_run = session.get(StageRun, stage_run_id) + if stage_run is None: + raise RuntimeError(f"stage run disappeared: {stage_run_id}") + stage_run.status = status + stage_run.output_json = output or {} + stage_run.error_message = error_message + stage_run.finished_at = utc_now() + + +def _checkpoint( + context: StageContext, run_id: str, outputs: dict[str, Any] +) -> None: + with context.storage.session() as session: + run = session.get(PipelineRun, run_id) + if run is not None: + run.output_json = outputs + run.heartbeat_at = utc_now() + + +def _successful_stage_output( + context: StageContext, run_id: str, stage_number: int +) -> dict[str, Any] | None: + with context.storage.session() as session: + stage_run = session.scalar( + select(StageRun) + .where( + StageRun.pipeline_run_id == run_id, + StageRun.stage_number == stage_number, + StageRun.status == "succeeded", + ) + .order_by(StageRun.attempt.desc()) + .limit(1) + ) + return dict(stage_run.output_json) if stage_run is not None else None diff --git a/src/graphrag_platform/pipeline.py b/src/graphrag_platform/pipeline.py new file mode 100644 index 0000000..685ed51 --- /dev/null +++ b/src/graphrag_platform/pipeline.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from graphrag_platform.config import Settings +from graphrag_platform.llm import LLMClient +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import load_stage +from graphrag_platform.storage import Storage + + +class KnowledgeGraphPipeline: + """Compatibility facade over independently executable numbered stages.""" + + def __init__( + self, + settings: Settings, + *, + storage: Storage | None = None, + llm: LLMClient | None = None, + ) -> None: + self.context = StageContext.create(settings, storage=storage, llm=llm) + + @property + def settings(self) -> Settings: + return self.context.settings + + @property + def storage(self) -> Storage: + return self.context.storage + + def initialize(self) -> None: + self.context.initialize() + + def stage_01_normalize( + self, *, input_jsonl: Path | None = None + ) -> dict[str, int]: + return load_stage(1).run(self.context, input_jsonl=input_jsonl) + + def stage_02_clean_and_chunk(self) -> dict[str, int]: + return load_stage(2).run(self.context) + + def stage_03_define_model( + self, model_file: Path | None = None + ) -> dict[str, int]: + return load_stage(3).run(self.context, model_file=model_file) + + def stage_04_extract(self) -> dict[str, int]: + return load_stage(4).run(self.context) + + def stage_05_reconcile(self) -> dict[str, int]: + return load_stage(5).run(self.context) + + def stage_06_build_graph(self) -> dict[str, int]: + return load_stage(6).run(self.context) + + def stage_07_build_communities(self) -> dict[str, int]: + return load_stage(7).run(self.context) + + def stage_08_build_retrieval_index(self) -> dict[str, int]: + return load_stage(8).run(self.context) + + def stage_09_retrieve( + self, question: str, *, top_k: int | None = None + ) -> dict[str, Any]: + return load_stage(9).run(self.context, question=question, top_k=top_k) + + def stage_10_answer( + self, question: str, *, top_k: int | None = None + ) -> dict[str, Any]: + return load_stage(10).run(self.context, question=question, top_k=top_k) diff --git a/src/graphrag_platform/runtime.py b/src/graphrag_platform/runtime.py new file mode 100644 index 0000000..63e2676 --- /dev/null +++ b/src/graphrag_platform/runtime.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from graphrag_platform.config import Settings +from graphrag_platform.llm import LLMClient, OpenAICompatibleClient +from graphrag_platform.storage import Storage + + +@dataclass +class StageContext: + settings: Settings + storage: Storage + _llm: LLMClient | None = None + + @classmethod + def create( + cls, + settings: Settings, + *, + storage: Storage | None = None, + llm: LLMClient | None = None, + ) -> StageContext: + return cls( + settings=settings, + storage=storage or Storage(settings.database_url), + _llm=llm, + ) + + @property + def llm(self) -> LLMClient: + if self._llm is None: + self._llm = OpenAICompatibleClient(self.settings) + return self._llm + + def initialize(self) -> None: + if self.settings.auto_create_schema: + self.storage.create_schema() + else: + self.storage.verify_schema() + (self.settings.artifact_dir / self.settings.corpus_name).mkdir( + parents=True, exist_ok=True + ) + + def for_corpus(self, corpus: str) -> StageContext: + return StageContext.create( + self.settings.model_copy(update={"corpus_name": corpus}), + storage=self.storage, + llm=self._llm, + ) diff --git a/src/graphrag_platform/sources.py b/src/graphrag_platform/sources.py new file mode 100644 index 0000000..e5b2557 --- /dev/null +++ b/src/graphrag_platform/sources.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import json +import re +from collections.abc import Iterator +from datetime import datetime +from pathlib import Path +from typing import Any, Protocol + +from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator +from sqlalchemy import MetaData, Table, create_engine, select + +_TABLE_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +class SourceRecord(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: str = "1.0" + external_id: str = Field(min_length=1, max_length=500) + corpus: str | None = Field(default=None, min_length=1, max_length=200) + url: HttpUrl + body: str = Field(min_length=1) + content_type: str = "text/html" + status_code: int = Field(default=200, ge=100, le=599) + retrieved_at: datetime + metadata: dict[str, Any] = Field(default_factory=dict) + + @field_validator("schema_version") + @classmethod + def require_supported_schema(cls, value: str) -> str: + if value != "1.0": + raise ValueError(f"unsupported source schema version: {value}") + return value + + +class SourceReader(Protocol): + def records(self) -> Iterator[SourceRecord]: ... + + +class JsonLinesSourceReader: + def __init__(self, path: Path) -> None: + self.path = path + + def records(self) -> Iterator[SourceRecord]: + with self.path.open("r", encoding="utf-8") as stream: + for line_number, line in enumerate(stream, start=1): + if not line.strip(): + continue + try: + payload = json.loads(line) + yield SourceRecord.model_validate(payload) + except (json.JSONDecodeError, ValueError) as error: + raise ValueError( + f"{self.path}:{line_number}: invalid source record: {error}" + ) from error + + +class PostgreSqlSourceReader: + def __init__(self, database_url: str, table_name: str) -> None: + if not database_url.startswith("postgresql"): + raise ValueError("SOURCE_DATABASE_URL must use PostgreSQL") + if not _TABLE_NAME_RE.fullmatch(table_name): + raise ValueError("SOURCE_TABLE_NAME must be an unqualified SQL identifier") + self.engine = create_engine(database_url, pool_pre_ping=True) + self.table_name = table_name + + def records(self) -> Iterator[SourceRecord]: + metadata = MetaData() + table = Table(self.table_name, metadata, autoload_with=self.engine) + required = { + "schema_version", + "external_id", + "url", + "body", + "retrieved_at", + } + missing = required - set(table.columns.keys()) + if missing: + raise ValueError( + f"source table {self.table_name} is missing columns: " + f"{', '.join(sorted(missing))}" + ) + with self.engine.connect() as connection: + for row in connection.execute(select(table).order_by(table.c.external_id)): + payload = { + key: value + for key, value in row._mapping.items() + if key in SourceRecord.model_fields + } + yield SourceRecord.model_validate(payload) + + +def source_reader( + *, + input_jsonl: Path | None, + source_database_url: str | None, + source_table_name: str, +) -> SourceReader: + if input_jsonl is not None and source_database_url: + raise ValueError("provide either --input-jsonl or SOURCE_DATABASE_URL, not both") + if input_jsonl is not None: + return JsonLinesSourceReader(input_jsonl) + if source_database_url: + return PostgreSqlSourceReader(source_database_url, source_table_name) + raise ValueError("provide --input-jsonl or configure SOURCE_DATABASE_URL") diff --git a/src/graphrag_platform/stage_loader.py b/src/graphrag_platform/stage_loader.py new file mode 100644 index 0000000..4a0a8c0 --- /dev/null +++ b/src/graphrag_platform/stage_loader.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from importlib import import_module +from types import ModuleType + +STAGE_MODULES = { + 1: "01_normalize_content", + 2: "02_clean_content", + 3: "03_define_knowledge_model", + 4: "04_extract_knowledge", + 5: "05_reconcile_knowledge", + 6: "06_build_knowledge_graph", + 7: "07_build_communities", + 8: "08_build_retrieval_index", + 9: "09_query_graphrag", + 10: "10_generate_answer", +} + + +def load_stage(stage: int) -> ModuleType: + try: + module_name = STAGE_MODULES[stage] + except KeyError as error: + raise ValueError(f"unsupported stage {stage}") from error + return import_module(module_name) diff --git a/src/graphrag_platform/storage.py b/src/graphrag_platform/storage.py new file mode 100644 index 0000000..b0d7e8e --- /dev/null +++ b/src/graphrag_platform/storage.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any, TypeVar + +from sqlalchemy import create_engine, delete, event, inspect, select, text +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session + +from graphrag_platform.models import ( + Base, + Chunk, + Community, + Document, + Edge, + Extraction, + KnowledgeModel, + Node, + RawDocument, + ResolvedEntity, + ResolvedRelationship, + SearchItem, +) + +ModelT = TypeVar("ModelT", bound=Base) + + +class Storage: + def __init__(self, database_url: str) -> None: + connect_args = {"check_same_thread": False} if database_url.startswith("sqlite") else {} + self.engine: Engine = create_engine( + database_url, + future=True, + pool_pre_ping=True, + connect_args=connect_args, + ) + if database_url.startswith("sqlite"): + event.listen(self.engine, "connect", self._enable_sqlite_foreign_keys) + + @staticmethod + def _enable_sqlite_foreign_keys( + dbapi_connection: Any, connection_record: Any + ) -> None: + del connection_record + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + + def create_schema(self) -> None: + if self.engine.dialect.name == "postgresql": + with self.engine.begin() as connection: + connection.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) + Base.metadata.create_all(self.engine) + + def verify_schema(self) -> None: + tables = set(inspect(self.engine).get_table_names()) + required = {"pipeline_runs", "stage_runs", "documents", "retrieval_items"} + missing = required - tables + if missing: + raise RuntimeError( + "database migrations are required; missing tables: " + f"{', '.join(sorted(missing))}" + ) + if self.engine.dialect.name == "postgresql": + with self.engine.connect() as connection: + extension = connection.scalar( + text( + "SELECT extname FROM pg_extension " + "WHERE extname = 'vector'" + ) + ) + if extension != "vector": + raise RuntimeError("database migrations are required: vector extension") + + @property + def supports_vector_search(self) -> bool: + return self.engine.dialect.name == "postgresql" + + def ensure_vector_index(self, dimensions: int) -> None: + if not self.supports_vector_search: + return + index_name = f"ix_retrieval_items_embedding_hnsw_{dimensions}" + statement = f""" + CREATE INDEX IF NOT EXISTS {index_name} + ON retrieval_items + USING hnsw ((embedding::vector({dimensions})) vector_cosine_ops) + WHERE vector_dims(embedding) = {dimensions} + """ + with self.engine.begin() as connection: + connection.execute(text(statement)) + + @contextmanager + def session(self) -> Iterator[Session]: + with Session(self.engine) as session: + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + + def upsert( + self, + session: Session, + model: type[ModelT], + identity: dict[str, Any], + values: dict[str, Any], + ) -> ModelT: + instance = session.scalar(select(model).filter_by(**identity)) + if instance is None: + instance = model(**identity, **values) + session.add(instance) + else: + for key, value in values.items(): + setattr(instance, key, value) + session.flush() + return instance + + def clear_from_stage(self, corpus: str, stage: int) -> None: + stage_tables: list[tuple[int, type[Base]]] = [ + (8, SearchItem), + (7, Community), + (6, Edge), + (6, Node), + (5, ResolvedRelationship), + (5, ResolvedEntity), + (4, Extraction), + (2, Chunk), + (1, RawDocument), + (1, Document), + ] + with self.session() as session: + for table_stage, model in stage_tables: + if table_stage >= stage: + session.execute(delete(model).where(model.corpus == corpus)) + if stage <= 3: + session.execute( + delete(KnowledgeModel).where(KnowledgeModel.corpus == corpus) + ) \ No newline at end of file diff --git a/src/graphrag_platform/worker.py b/src/graphrag_platform/worker.py new file mode 100644 index 0000000..cdb5e10 --- /dev/null +++ b/src/graphrag_platform/worker.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import argparse +import logging +import time + +from graphrag_platform.config import Settings +from graphrag_platform.observability import configure_logging +from graphrag_platform.orchestrator import ( + RunDeferredError, + claim_next_run, + execute_run, + recover_stale_runs, +) +from graphrag_platform.runtime import StageContext + +LOGGER = logging.getLogger(__name__) + + +def work_once(context: StageContext) -> bool: + run_id = claim_next_run(context) + if run_id is None: + return False + try: + execute_run(context, run_id) + except RunDeferredError: + LOGGER.info( + "Worker deferred a run for an active corpus", + extra={"event": "worker_job_deferred", "run_id": run_id}, + ) + except Exception: + LOGGER.exception( + "Worker execution failed", + extra={"event": "worker_job_failed", "run_id": run_id}, + ) + return True + + +def run_worker(context: StageContext, *, once: bool = False) -> None: + context.initialize() + recover_stale_runs(context) + while True: + worked = work_once(context) + if once: + return + if not worked: + time.sleep(context.settings.worker_poll_seconds) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run the KnowledgeGraph job worker.") + parser.add_argument("--env-file", default=".env") + parser.add_argument("--once", action="store_true") + args = parser.parse_args() + settings = Settings(_env_file=args.env_file) + configure_logging(settings.log_level) + run_worker(StageContext.create(settings), once=args.once) + + +if __name__ == "__main__": + main() diff --git a/src/templates/knowledge_model.json b/src/templates/knowledge_model.json new file mode 100644 index 0000000..81209cb --- /dev/null +++ b/src/templates/knowledge_model.json @@ -0,0 +1,33 @@ +{ + "entity_types": [ + "Person", + "Organization", + "Product", + "Concept", + "Event", + "Location" + ], + "relationship_types": [ + "AFFILIATED_WITH", + "CREATED", + "DEPENDS_ON", + "MENTIONS", + "PART_OF", + "RELATED_TO" + ], + "node_template": { + "canonical_name": "string", + "entity_type": "one configured entity type", + "description": "source-grounded string", + "aliases": [ + "string" + ] + }, + "edge_template": { + "source": "entity name", + "target": "entity name", + "relationship_type": "one configured relationship type", + "description": "source-grounded string" + }, + "extraction_instructions": "Extract only entities and relationships explicitly supported by the supplied text." +} diff --git a/src/templates/mcp_documents.jsonl b/src/templates/mcp_documents.jsonl new file mode 100644 index 0000000..0d5ff8f --- /dev/null +++ b/src/templates/mcp_documents.jsonl @@ -0,0 +1 @@ +{"schema_version":"1.0","external_id":"docs-example-article","corpus":"product-knowledge","url":"https://docs.example.com/article","content_type":"text/html","status_code":200,"retrieved_at":"2026-01-01T00:00:00Z","body":"Article title

Article title

Article text collected with Web-Access.

","metadata":{"collector":"Web-Access","stored_by":"Data-Store"}} diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..c3066e7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import hashlib +from collections.abc import Sequence +from typing import Any + + +class FakeLLM: + chat_model = "fake-chat-model" + embedding_model = "fake-embedding-model" + + def complete_json(self, system: str, user: str) -> dict[str, Any]: + if "community" in system.casefold(): + return { + "title": "Atlas product community", + "summary": "Acme created Atlas, a graph retrieval product.", + } + return { + "entities": [ + { + "canonical_name": "Acme", + "entity_type": "Organization", + "description": "Acme is a technology organization.", + "aliases": ["Acme Corp"], + }, + { + "canonical_name": "Atlas", + "entity_type": "Product", + "description": "Atlas is a graph retrieval product.", + "aliases": [], + }, + ], + "relationships": [ + { + "source": "Acme", + "target": "Atlas", + "relationship_type": "CREATED", + "description": "Acme created Atlas.", + } + ], + } + + def complete_text(self, system: str, user: str) -> str: + return "Acme created Atlas. [S1]" + + def embed(self, texts: Sequence[str]) -> list[list[float]]: + vectors: list[list[float]] = [] + for text in texts: + normalized = text.casefold() + digest = hashlib.sha256(normalized.encode("utf-8")).digest() + vectors.append( + [ + float("atlas" in normalized), + float("acme" in normalized), + digest[0] / 255, + ] + ) + return vectors diff --git a/tests/fixtures/external_documents.jsonl b/tests/fixtures/external_documents.jsonl new file mode 100644 index 0000000..134ff86 --- /dev/null +++ b/tests/fixtures/external_documents.jsonl @@ -0,0 +1,3 @@ +{"schema_version":"1.0","external_id":"atlas-overview","corpus":"acceptance-corpus","url":"https://docs.example/atlas","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:00:00Z","body":"Atlas Overview

Atlas

Acme Corporation created Atlas. Atlas provides graph retrieval for knowledge applications.

","metadata":{"collector":"Web-Access","stored_by":"Data-Store","tags":["product","graph"]}} +{"schema_version":"1.0","external_id":"atlas-architecture","corpus":"acceptance-corpus","url":"https://docs.example/atlas/architecture","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:01:00Z","body":"Atlas Architecture

Architecture

Acme Corp designed Atlas to combine a knowledge graph with vector retrieval. Atlas depends on PostgreSQL and PgVector.

","metadata":{"collector":"Web-Access","stored_by":"Data-Store","description":"Contains commas, \"quotes\", and aliases."}} +{"schema_version":"1.0","external_id":"atlas-overview-copy","corpus":"acceptance-corpus","url":"https://mirror.example/atlas","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:02:00Z","body":"Atlas Overview

Atlas

Acme Corporation created Atlas. Atlas provides graph retrieval for knowledge applications.

","metadata":{"collector":"Web-Access","stored_by":"Data-Store","duplicate_candidate":true}} diff --git a/tests/integration/test_api_worker.py b/tests/integration/test_api_worker.py new file mode 100644 index 0000000..c1b6424 --- /dev/null +++ b/tests/integration/test_api_worker.py @@ -0,0 +1,284 @@ +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone +from pathlib import Path +from uuid import uuid4 + +import jwt +from fastapi.testclient import TestClient + +from graphrag_platform.api import create_app +from graphrag_platform.auth import Permission +from graphrag_platform.config import Settings +from graphrag_platform.runtime import StageContext +from graphrag_platform.worker import work_once + +from conftest import FakeLLM + + +def _token(settings: Settings, mask: int) -> str: + now = datetime.now(timezone.utc) + return jwt.encode( + { + "iss": settings.internal_jwt_issuer, + "aud": settings.internal_jwt_audience, + "sub": "dev-test-client", + "iat": now, + "exp": now + timedelta(minutes=10), + "jti": str(uuid4()), + "features": {settings.internal_feature_name: mask}, + }, + settings.internal_jwt_secret, + algorithm="HS256", + ) + + +def test_bff_contract_and_worker_execute_durable_job(tmp_path: Path) -> None: + source = tmp_path / "source.jsonl" + source.write_text( + json.dumps( + { + "schema_version": "1.0", + "external_id": "atlas", + "corpus": "api-corpus", + "url": "https://docs.example/atlas", + "retrieved_at": "2026-10-13T18:00:00Z", + "body": ( + "Atlas
" + "Acme created Atlas for graph retrieval.
" + ), + } + ) + + "\n", + encoding="utf-8", + ) + settings = Settings( + _env_file=None, + database_url=f"sqlite:///{tmp_path / 'api.db'}", + auto_create_schema=True, + artifact_dir=tmp_path / "artifacts", + llm_embedding_dimensions=3, + internal_jwt_secret="dev-test-secret-at-least-32-bytes-long", + ) + app = create_app(settings, llm=FakeLLM()) + full_token = _token( + settings, + int(Permission.READ | Permission.ADD | Permission.EXECUTE), + ) + + with TestClient(app) as client: + assert client.get("/internal/knowledge-graph/health").status_code == 200 + assert ( + client.post( + "/internal/knowledge-graph/jobs", + json={ + "corpus": "api-corpus", + "from_stage": 1, + "through_stage": 8, + "input_jsonl": str(source), + }, + ).status_code + == 401 + ) + execute_only = _token(settings, int(Permission.EXECUTE)) + assert ( + client.post( + "/internal/knowledge-graph/jobs", + headers={"Authorization": f"Bearer {execute_only}"}, + json={ + "corpus": "api-corpus", + "from_stage": 1, + "through_stage": 8, + "input_jsonl": str(source), + }, + ).status_code + == 403 + ) + queued = client.post( + "/internal/knowledge-graph/jobs", + headers={"Authorization": f"Bearer {full_token}"}, + json={ + "corpus": "api-corpus", + "from_stage": 1, + "through_stage": 8, + "input_jsonl": str(source), + }, + ) + assert queued.status_code == 202 + job_id = queued.json()["job_id"] + + assert work_once(StageContext.create(settings, llm=FakeLLM())) + + status_response = client.get( + f"/internal/knowledge-graph/jobs/{job_id}", + headers={"Authorization": f"Bearer {full_token}"}, + ) + assert status_response.status_code == 200 + assert status_response.json()["status"] == "succeeded" + assert len(status_response.json()["stages"]) == 8 + + answer = client.post( + "/internal/knowledge-graph/answer", + headers={"Authorization": f"Bearer {full_token}"}, + json={"corpus": "api-corpus", "question": "Who created Atlas?"}, + ) + assert answer.status_code == 200 + assert answer.json()["answer"] == "Acme created Atlas. [S1]" + + +def test_failed_job_can_be_repaired_and_retried(tmp_path: Path) -> None: + source = tmp_path / "retry-source.jsonl" + source.write_text( + json.dumps( + { + "schema_version": "2.0", + "external_id": "retry-record", + "corpus": "retry-corpus", + "url": "https://docs.example/retry", + "retrieved_at": "2026-10-13T18:00:00Z", + "body": "invalid schema", + } + ) + + "\n", + encoding="utf-8", + ) + settings = Settings( + _env_file=None, + database_url=f"sqlite:///{tmp_path / 'retry.db'}", + auto_create_schema=True, + artifact_dir=tmp_path / "artifacts", + llm_embedding_dimensions=3, + internal_jwt_secret="dev-test-secret-at-least-32-bytes-long", + ) + app = create_app(settings, llm=FakeLLM()) + token = _token( + settings, + int(Permission.READ | Permission.ADD | Permission.EXECUTE), + ) + headers = {"Authorization": f"Bearer {token}"} + + with TestClient(app) as client: + queued = client.post( + "/internal/knowledge-graph/jobs", + headers=headers, + json={ + "corpus": "retry-corpus", + "from_stage": 1, + "through_stage": 1, + "input_jsonl": str(source), + }, + ) + job_id = queued.json()["job_id"] + assert work_once(StageContext.create(settings, llm=FakeLLM())) + assert ( + client.get( + f"/internal/knowledge-graph/jobs/{job_id}", + headers=headers, + ).json()["status"] + == "failed" + ) + + source.write_text( + json.dumps( + { + "schema_version": "1.0", + "external_id": "retry-record", + "corpus": "retry-corpus", + "url": "https://docs.example/retry", + "retrieved_at": "2026-10-13T18:00:00Z", + "content_type": "text/plain", + "body": "Corrected source content.", + } + ) + + "\n", + encoding="utf-8", + ) + retried = client.post( + f"/internal/knowledge-graph/jobs/{job_id}/retry", + headers=headers, + ) + assert retried.status_code == 202 + assert work_once(StageContext.create(settings, llm=FakeLLM())) + status_response = client.get( + f"/internal/knowledge-graph/jobs/{job_id}", + headers=headers, + ) + assert status_response.json()["status"] == "succeeded" + assert status_response.json()["attempt"] == 2 + + +def test_retry_resumes_after_last_successful_stage(tmp_path: Path) -> None: + source = tmp_path / "resume-source.jsonl" + source.write_text( + json.dumps( + { + "schema_version": "1.0", + "external_id": "resume-record", + "corpus": "resume-corpus", + "url": "https://docs.example/resume", + "retrieved_at": "2026-10-13T18:00:00Z", + "content_type": "text/plain", + "body": "Acme created Atlas for graph retrieval.", + } + ) + + "\n", + encoding="utf-8", + ) + model_file = tmp_path / "model.json" + model_file.write_text("[]", encoding="utf-8") + settings = Settings( + _env_file=None, + database_url=f"sqlite:///{tmp_path / 'resume.db'}", + auto_create_schema=True, + artifact_dir=tmp_path / "artifacts", + llm_embedding_dimensions=3, + internal_jwt_secret="dev-test-secret-at-least-32-bytes-long", + ) + app = create_app(settings, llm=FakeLLM()) + token = _token( + settings, + int(Permission.READ | Permission.ADD | Permission.EXECUTE), + ) + headers = {"Authorization": f"Bearer {token}"} + + with TestClient(app) as client: + queued = client.post( + "/internal/knowledge-graph/jobs", + headers=headers, + json={ + "corpus": "resume-corpus", + "from_stage": 1, + "through_stage": 3, + "input_jsonl": str(source), + "model_file": str(model_file), + }, + ) + job_id = queued.json()["job_id"] + assert work_once(StageContext.create(settings, llm=FakeLLM())) + failed = client.get( + f"/internal/knowledge-graph/jobs/{job_id}", headers=headers + ).json() + assert failed["status"] == "failed" + + model_file.write_text( + json.dumps(settings.model_definition()), + encoding="utf-8", + ) + assert ( + client.post( + f"/internal/knowledge-graph/jobs/{job_id}/retry", + headers=headers, + ).status_code + == 202 + ) + assert work_once(StageContext.create(settings, llm=FakeLLM())) + succeeded = client.get( + f"/internal/knowledge-graph/jobs/{job_id}", headers=headers + ).json() + + assert succeeded["status"] == "succeeded" + stage_numbers = [row["stage"] for row in succeeded["stages"]] + assert stage_numbers.count(1) == 1 + assert stage_numbers.count(2) == 1 + assert stage_numbers.count(3) == 2 diff --git a/tests/integration/test_full_pipeline.py b/tests/integration/test_full_pipeline.py new file mode 100644 index 0000000..6cb007b --- /dev/null +++ b/tests/integration/test_full_pipeline.py @@ -0,0 +1,69 @@ +import json +from pathlib import Path + +from sqlalchemy import func, select + +from graphrag_platform.config import Settings +from graphrag_platform.models import Community, Edge, Node, SearchItem +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import load_stage + +from conftest import FakeLLM + + +def test_all_ten_stages_build_and_query_graph(tmp_path: Path) -> None: + input_file = tmp_path / "documents.jsonl" + input_file.write_text( + json.dumps( + { + "schema_version": "1.0", + "external_id": "atlas-doc", + "corpus": "atlas-corpus", + "url": "https://docs.example/atlas", + "retrieved_at": "2026-10-13T18:00:00Z", + "body": ( + "Atlas
" + "

Atlas

Acme created Atlas. Atlas provides graph " + "retrieval for knowledge applications.

" + ), + "metadata": {"collector": "Web-Access", "stored_by": "Data-Store"}, + } + ) + + "\n", + encoding="utf-8", + ) + settings = Settings( + _env_file=None, + database_url=f"sqlite:///{tmp_path / 'graph.db'}", + auto_create_schema=True, + artifact_dir=tmp_path / "artifacts", + corpus_name="atlas-corpus", + chunk_size=300, + chunk_overlap=30, + retrieval_top_k=5, + llm_embedding_dimensions=3, + ) + context = StageContext.create(settings, llm=FakeLLM()) + + assert load_stage(1).run(context, input_jsonl=input_file)["normalized"] == 1 + assert load_stage(2).run(context)["chunks"] == 1 + assert load_stage(3).run(context)["entity_types"] == 6 + assert load_stage(4).run(context)["extracted_chunks"] == 1 + assert load_stage(5).run(context) == {"entities": 2, "relationships": 1} + assert load_stage(6).run(context) == {"nodes": 2, "edges": 1} + assert load_stage(7).run(context)["communities"] == 1 + assert load_stage(8).run(context)["indexed_items"] == 5 + + retrieval = load_stage(9).run(context, question="Who created Atlas?") + response = load_stage(10).run(context, question="Who created Atlas?") + + assert retrieval["context"] + assert retrieval["graph_context"][0]["relationship"] == "CREATED" + assert retrieval["sources"][0]["url"] == "https://docs.example/atlas" + assert response["answer"] == "Acme created Atlas. [S1]" + assert response["sources"][0]["citation"] == "S1" + with context.storage.session() as session: + assert session.scalar(select(func.count()).select_from(Node)) == 2 + assert session.scalar(select(func.count()).select_from(Edge)) == 1 + assert session.scalar(select(func.count()).select_from(Community)) == 1 + assert session.scalar(select(func.count()).select_from(SearchItem)) == 5 diff --git a/tests/integration/test_pgvector_postgres.py b/tests/integration/test_pgvector_postgres.py new file mode 100644 index 0000000..33ede97 --- /dev/null +++ b/tests/integration/test_pgvector_postgres.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +from sqlalchemy import text + +from graphrag_platform.config import Settings +from graphrag_platform.models import PipelineRun +from graphrag_platform.orchestrator import ( + RunDeferredError, + claim_next_run, + execute_run, + submit_run, +) +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import load_stage + +from conftest import FakeLLM + +POSTGRES_URL = os.getenv("TEST_POSTGRES_URL") + + +@pytest.mark.skipif( + not POSTGRES_URL, + reason="TEST_POSTGRES_URL is required for the PostgreSQL integration test", +) +def test_pgvector_index_and_graph_retrieval( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + settings = Settings( + _env_file=None, + database_url=POSTGRES_URL, + source_database_url=POSTGRES_URL, + source_table_name="external_documents", + artifact_dir=tmp_path / "artifacts", + corpus_name="pgvector-integration", + chunk_size=300, + chunk_overlap=30, + retrieval_top_k=5, + llm_embedding_dimensions=3, + ) + monkeypatch.setenv("DATABASE_URL", POSTGRES_URL) + from alembic import command + from alembic.config import Config + + alembic_config = Config("alembic.ini") + command.downgrade(alembic_config, "base") + command.upgrade(alembic_config, "head") + context = StageContext.create(settings, llm=FakeLLM()) + context.storage.clear_from_stage(settings.corpus_name, 1) + with context.storage.engine.begin() as connection: + connection.execute(text("DROP TABLE IF EXISTS external_documents")) + connection.execute( + text( + """ + CREATE TABLE external_documents ( + id bigserial PRIMARY KEY, + external_id text NOT NULL UNIQUE, + schema_version text NOT NULL, + corpus text, + url text NOT NULL, + body text NOT NULL, + content_type text NOT NULL, + status_code integer NOT NULL, + retrieved_at timestamptz NOT NULL, + metadata jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + ) + connection.execute( + text( + """ + INSERT INTO external_documents ( + external_id, schema_version, corpus, url, body, + content_type, status_code, retrieved_at, metadata + ) VALUES ( + 'atlas-doc', '1.0', 'pgvector-integration', + 'https://docs.example/atlas', + 'Atlas

Atlas

Acme created Atlas. Atlas provides graph retrieval for knowledge applications.

', + 'text/html', 200, '2026-10-13T18:00:00Z', + '{"collector":"Web-Access","stored_by":"Data-Store"}'::jsonb + ) + """ + ) + ) + + load_stage(1).run(context) + load_stage(2).run(context) + load_stage(3).run(context) + load_stage(4).run(context) + load_stage(5).run(context) + load_stage(6).run(context) + load_stage(7).run(context) + load_stage(8).run(context) + result = load_stage(9).run(context, question="Who created Atlas?") + + assert result["context"] + assert result["graph_context"][0]["relationship"] == "CREATED" + assert result["sources"][0]["url"] == "https://docs.example/atlas" + with context.storage.engine.connect() as connection: + assert connection.scalar( + text("SELECT extversion FROM pg_extension WHERE extname = 'vector'") + ) + index_definition = connection.scalar( + text( + "SELECT indexdef FROM pg_indexes " + "WHERE indexname = 'ix_retrieval_items_embedding_hnsw_3'" + ) + ) + assert index_definition is not None + assert "USING hnsw" in index_definition + + run_id = submit_run( + context, + from_stage=2, + through_stage=2, + payload={}, + requested_by="integration-test", + correlation_id="pg-lock-test", + permission_mask=31, + ) + with context.storage.engine.connect() as lock_connection: + assert lock_connection.scalar( + text("SELECT pg_try_advisory_lock(hashtext(:key))"), + {"key": "knowledgegraph:pgvector-integration"}, + ) + with pytest.raises(RunDeferredError): + execute_run(context, run_id) + lock_connection.execute( + text("SELECT pg_advisory_unlock(hashtext(:key))"), + {"key": "knowledgegraph:pgvector-integration"}, + ) + with context.storage.session() as session: + assert session.get(PipelineRun, run_id).status == "queued" + assert claim_next_run(context) == run_id + assert execute_run(context, run_id)["2"]["chunks"] == 1 diff --git a/tests/performance/test_vector_query_load.py b/tests/performance/test_vector_query_load.py new file mode 100644 index 0000000..8b4d20c --- /dev/null +++ b/tests/performance/test_vector_query_load.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import os +import time + +import pytest + +from graphrag_platform.config import Settings +from graphrag_platform.models import SearchItem +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import load_stage + +from conftest import FakeLLM + +POSTGRES_URL = os.getenv("TEST_POSTGRES_URL") +RUN_PERFORMANCE_TESTS = os.getenv("RUN_PERFORMANCE_TESTS") == "1" + + +@pytest.mark.performance +@pytest.mark.skipif( + not POSTGRES_URL or not RUN_PERFORMANCE_TESTS, + reason="Set TEST_POSTGRES_URL and RUN_PERFORMANCE_TESTS=1", +) +def test_vector_query_latency_at_configured_load() -> None: + item_count = int(os.getenv("PERF_VECTOR_ITEM_COUNT", "10000")) + maximum_seconds = float(os.getenv("PERF_MAX_QUERY_SECONDS", "2.0")) + settings = Settings( + _env_file=None, + database_url=POSTGRES_URL, + corpus_name="performance-corpus", + llm_embedding_dimensions=3, + retrieval_top_k=10, + ) + context = StageContext.create(settings, llm=FakeLLM()) + context.initialize() + with context.storage.session() as session: + session.query(SearchItem).filter_by(corpus=settings.corpus_name).delete() + session.add_all( + SearchItem( + corpus=settings.corpus_name, + item_type="benchmark", + item_key=str(index), + text=f"Benchmark graph item {index}", + source_url=None, + metadata_json={}, + embedding=[ + float(index % 2), + float((index // 2) % 2), + (index % 255) / 255, + ], + embedding_model=FakeLLM.embedding_model, + ) + for index in range(item_count) + ) + context.storage.ensure_vector_index(3) + + started = time.perf_counter() + result = load_stage(9).run(context, question="Atlas graph retrieval", top_k=10) + elapsed = time.perf_counter() - started + + assert len(result["context"]) == 10 + assert elapsed <= maximum_seconds, ( + f"vector query took {elapsed:.3f}s; threshold is {maximum_seconds:.3f}s" + ) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 0000000..4546db8 --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,25 @@ +from pathlib import Path + +from graphrag_platform.config import Settings + + +def test_settings_parse_runtime_templates(monkeypatch) -> None: + monkeypatch.setenv("ENTITY_TYPES", "Company,Product") + monkeypatch.setenv("RELATIONSHIP_TYPES", "CREATED,DEPENDS_ON") + monkeypatch.setenv("NODE_TEMPLATE", '{"name":"string"}') + + settings = Settings(_env_file=None) + + assert settings.entity_types == ["Company", "Product"] + assert settings.relationship_types == ["CREATED", "DEPENDS_ON"] + assert settings.node_template == {"name": "string"} + assert settings.database_url.startswith("postgresql+psycopg://") + + +def test_sqlite_is_available_as_explicit_local_override(tmp_path: Path) -> None: + settings = Settings( + _env_file=None, + database_url=f"sqlite:///{tmp_path / 'graph.db'}", + ) + + assert settings.database_url.startswith("sqlite:///") diff --git a/tests/unit/test_llm.py b/tests/unit/test_llm.py new file mode 100644 index 0000000..8933b05 --- /dev/null +++ b/tests/unit/test_llm.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import httpx + +from graphrag_platform.config import Settings +from graphrag_platform.llm import OpenAICompatibleClient + + +def test_llm_retries_rate_limit_and_honors_retry_after() -> None: + requests = 0 + delays: list[float] = [] + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal requests + requests += 1 + if requests == 1: + return httpx.Response( + 429, + headers={"Retry-After": "0.25"}, + json={"error": "rate limited"}, + request=request, + ) + return httpx.Response( + 200, + json={"choices": [{"message": {"content": "answer"}}]}, + request=request, + ) + + settings = Settings( + _env_file=None, + llm_api_key="test-key", + llm_max_retries=2, + ) + client = OpenAICompatibleClient( + settings, + transport=httpx.MockTransport(handler), + sleeper=delays.append, + jitter=lambda: 0, + ) + + assert client.complete_text("system", "user") == "answer" + assert requests == 2 + assert delays == [0.25] + + +def test_llm_does_not_retry_authentication_failure() -> None: + requests = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal requests + requests += 1 + return httpx.Response(401, json={"error": "unauthorized"}, request=request) + + client = OpenAICompatibleClient( + Settings(_env_file=None, llm_api_key="bad-key", llm_max_retries=3), + transport=httpx.MockTransport(handler), + sleeper=lambda _: None, + ) + + try: + client.complete_text("system", "user") + except httpx.HTTPStatusError: + pass + else: + raise AssertionError("Expected an HTTPStatusError") + assert requests == 1 diff --git a/tests/unit/test_pipeline.py b/tests/unit/test_pipeline.py new file mode 100644 index 0000000..c77d029 --- /dev/null +++ b/tests/unit/test_pipeline.py @@ -0,0 +1,113 @@ +from pathlib import Path + +import pytest +from sqlalchemy import select + +from graphrag_platform.config import Settings +from graphrag_platform.models import Chunk, Document, Extraction, ResolvedRelationship +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import load_stage + +from conftest import FakeLLM + + +def test_chunk_overlap_must_be_smaller_than_chunk_size(tmp_path: Path) -> None: + settings = Settings( + _env_file=None, + database_url=f"sqlite:///{tmp_path / 'graph.db'}", + auto_create_schema=True, + artifact_dir=tmp_path / "artifacts", + chunk_size=200, + chunk_overlap=200, + ) + stage = load_stage(2) + + with pytest.raises(ValueError, match="CHUNK_OVERLAP"): + stage.chunk_text("word " * 100, size=200, overlap=200) + + +def test_reconciliation_keeps_cross_chunk_relationship(tmp_path: Path) -> None: + settings = Settings( + _env_file=None, + database_url=f"sqlite:///{tmp_path / 'graph.db'}", + auto_create_schema=True, + artifact_dir=tmp_path / "artifacts", + corpus_name="unit", + ) + context = StageContext.create(settings, llm=FakeLLM()) + context.initialize() + with context.storage.session() as session: + document = Document( + corpus="unit", + url="https://example.test/source", + text="Acme created Atlas.", + content_hash="document", + metadata_json={}, + ) + session.add(document) + session.flush() + first = Chunk( + document_id=document.id, + corpus="unit", + ordinal=0, + text="Acme created", + content_hash="first", + ) + second = Chunk( + document_id=document.id, + corpus="unit", + ordinal=1, + text="Atlas", + content_hash="second", + ) + session.add_all([first, second]) + session.flush() + session.add_all( + [ + Extraction( + chunk_id=first.id, + corpus="unit", + model_name="test", + payload_json={ + "entities": [ + { + "canonical_name": "Acme", + "entity_type": "Organization", + "aliases": [], + } + ], + "relationships": [ + { + "source": "Acme", + "target": "Atlas", + "relationship_type": "CREATED", + } + ], + }, + ), + Extraction( + chunk_id=second.id, + corpus="unit", + model_name="test", + payload_json={ + "entities": [ + { + "canonical_name": "Atlas", + "entity_type": "Product", + "aliases": [], + } + ], + "relationships": [], + }, + ), + ] + ) + + result = load_stage(5).run(context) + + assert result == {"entities": 2, "relationships": 1} + with context.storage.session() as session: + relationship = session.scalar(select(ResolvedRelationship)) + assert relationship is not None + assert relationship.source_key == "acme" + assert relationship.target_key == "atlas" diff --git a/tests/unit/test_sources.py b/tests/unit/test_sources.py new file mode 100644 index 0000000..462b244 --- /dev/null +++ b/tests/unit/test_sources.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from graphrag_platform.sources import JsonLinesSourceReader + + +def test_jsonl_source_contract_preserves_nested_metadata(tmp_path: Path) -> None: + source = tmp_path / "source.jsonl" + source.write_text( + json.dumps( + { + "schema_version": "1.0", + "external_id": "record-1", + "corpus": "test", + "url": "https://example.test/article", + "body": "
Text, with commas and \"quotes\".
", + "retrieved_at": "2026-10-13T18:00:00Z", + "metadata": {"tags": ["one", "two"], "nested": {"value": 1}}, + } + ) + + "\n", + encoding="utf-8", + ) + + records = list(JsonLinesSourceReader(source).records()) + + assert records[0].external_id == "record-1" + assert records[0].metadata["tags"] == ["one", "two"] + assert '"quotes"' in records[0].body + + +def test_jsonl_source_contract_rejects_unknown_schema(tmp_path: Path) -> None: + source = tmp_path / "source.jsonl" + source.write_text( + json.dumps( + { + "schema_version": "2.0", + "external_id": "record-1", + "url": "https://example.test/article", + "body": "text", + "retrieved_at": "2026-10-13T18:00:00Z", + } + ) + + "\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="unsupported source schema"): + list(JsonLinesSourceReader(source).records())