diff --git a/.dockerignore b/.dockerignore
new file mode 100644
index 0000000..27d2442
--- /dev/null
+++ b/.dockerignore
@@ -0,0 +1,13 @@
+.env
+.git
+.github
+.venv
+__pycache__
+*.py[cod]
+.pytest_cache
+.coverage
+artifacts
+build
+dist
+*.egg-info
+tests
diff --git a/.env.example b/.env.example
new file mode 100644
index 0000000..5231978
--- /dev/null
+++ b/.env.example
@@ -0,0 +1,47 @@
+# Storage. Replace the host, user, password, and database for each environment.
+POSTGRES_PASSWORD=knowledgegraph-dev
+POSTGRES_PORT=5432
+API_PORT=8080
+DATABASE_URL=postgresql+psycopg://knowledgegraph_app:knowledgegraph-dev@localhost:5432/knowledgegraph
+SOURCE_DATABASE_URL=
+SOURCE_TABLE_NAME=external_documents
+ANALYTICS_DATABASE_URL=
+AUTO_CREATE_SCHEMA=false
+
+# External handoff and corpus.
+CORPUS_NAME=product-knowledge
+
+# Runtime model templates. JSON files passed with --model-file override these values.
+ENTITY_TYPES=Person,Organization,Product,Concept,Event,Location
+RELATIONSHIP_TYPES=AFFILIATED_WITH,CREATED,DEPENDS_ON,MENTIONS,PART_OF,RELATED_TO
+NODE_TEMPLATE={"canonical_name":"string","entity_type":"one configured entity type","description":"source-grounded string","aliases":["string"]}
+EDGE_TEMPLATE={"source":"entity name","target":"entity name","relationship_type":"one configured relationship type","description":"source-grounded string"}
+EXTRACTION_INSTRUCTIONS=Extract only entities and relationships explicitly supported by the supplied text.
+
+# OpenAI-compatible chat and embedding provider.
+LLM_BASE_URL=https://api.openai.com/v1
+LLM_API_KEY=replace-with-provider-key
+LLM_CHAT_MODEL=gpt-4.1-mini
+LLM_EMBEDDING_MODEL=text-embedding-3-small
+LLM_EMBEDDING_DIMENSIONS=1536
+LLM_TIMEOUT_SECONDS=60
+LLM_MAX_RETRIES=4
+LLM_RETRY_INITIAL_SECONDS=1
+LLM_RETRY_MAX_SECONDS=30
+
+# Processing and retrieval.
+CHUNK_SIZE=1200
+CHUNK_OVERLAP=150
+COMMUNITY_RESOLUTION=1.0
+RETRIEVAL_TOP_K=8
+RETRIEVAL_GRAPH_HOPS=2
+RETRIEVAL_GRAPH_LIMIT=50
+RETRIEVAL_HNSW_EF_SEARCH=100
+LOG_LEVEL=INFO
+INTERNAL_JWT_SECRET=replace-with-a-dev-test-secret-at-least-32-bytes
+INTERNAL_JWT_ISSUER=knowledgegraph-bff
+INTERNAL_JWT_AUDIENCE=knowledgegraph-service
+INTERNAL_FEATURE_NAME=KnowledgeGraph
+WORKER_POLL_SECONDS=2
+WORKER_STALE_RUN_SECONDS=3600
+ARTIFACT_DIR=artifacts
\ No newline at end of file
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
new file mode 100644
index 0000000..3207bca
--- /dev/null
+++ b/.github/workflows/ci.yml
@@ -0,0 +1,56 @@
+name: CI
+
+on:
+ push:
+ pull_request:
+
+jobs:
+ test:
+ runs-on: ubuntu-latest
+ services:
+ postgres:
+ image: pgvector/pgvector:pg16
+ env:
+ POSTGRES_USER: knowledgegraph_app
+ POSTGRES_PASSWORD: test-password
+ POSTGRES_DB: knowledgegraph
+ ports:
+ - 5432:5432
+ options: >-
+ --health-cmd "pg_isready -U knowledgegraph_app -d knowledgegraph"
+ --health-interval 5s
+ --health-timeout 5s
+ --health-retries 20
+
+ env:
+ DATABASE_URL: postgresql+psycopg://knowledgegraph_app:test-password@localhost:5432/knowledgegraph
+ TEST_POSTGRES_URL: postgresql+psycopg://knowledgegraph_app:test-password@localhost:5432/knowledgegraph
+
+ steps:
+ - uses: actions/checkout@v4
+
+ - uses: actions/setup-python@v5
+ with:
+ python-version: "3.12"
+ cache: pip
+
+ - name: Install
+ run: python -m pip install -e ".[test]"
+
+ - name: Compile
+ run: python -m compileall -q src tests migrations
+
+ - name: Check dependencies
+ run: python -m pip check
+
+ - name: Run portable tests
+ run: python -m pytest -q --ignore=tests/integration/test_pgvector_postgres.py
+
+ - name: Run migration and PgVector integration
+ run: python -m pytest tests/integration/test_pgvector_postgres.py -q
+
+ - name: Validate Compose
+ run: docker compose config --quiet
+
+ - name: Build application image
+ run: docker build -f Containerfile -t knowledgegraph:test .
diff --git a/.gitignore b/.gitignore
index d5dca0c..255642b 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,414 +1,12 @@
-# ---> VisualStudio
-## Ignore Visual Studio temporary files, build results, and
-## files generated by popular Visual Studio add-ons.
-##
-## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore
-
-# User-specific files
-*.rsuser
-*.suo
-*.user
-*.userosscache
-*.sln.docstates
-
-# User-specific files (MonoDevelop/Xamarin Studio)
-*.userprefs
-
-# Mono auto generated files
-mono_crash.*
-
-# Build results
-[Dd]ebug/
-[Dd]ebugPublic/
-[Rr]elease/
-[Rr]eleases/
-x64/
-x86/
-[Ww][Ii][Nn]32/
-[Aa][Rr][Mm]/
-[Aa][Rr][Mm]64/
-bld/
-[Bb]in/
-[Oo]bj/
-[Ll]og/
-[Ll]ogs/
-
-# Visual Studio 2015/2017 cache/options directory
-.vs/
-# Uncomment if you have tasks that create the project's static files in wwwroot
-#wwwroot/
-
-# Visual Studio 2017 auto generated files
-Generated\ Files/
-
-# MSTest test Results
-[Tt]est[Rr]esult*/
-[Bb]uild[Ll]og.*
-
-# NUnit
-*.VisualState.xml
-TestResult.xml
-nunit-*.xml
-
-# Build Results of an ATL Project
-[Dd]ebugPS/
-[Rr]eleasePS/
-dlldata.c
-
-# Benchmark Results
-BenchmarkDotNet.Artifacts/
-
-# .NET Core
-project.lock.json
-project.fragment.lock.json
-artifacts/
-
-# ASP.NET Scaffolding
-ScaffoldingReadMe.txt
-
-# StyleCop
-StyleCopReport.xml
-
-# Files built by Visual Studio
-*_i.c
-*_p.c
-*_h.h
-*.ilk
-*.meta
-*.obj
-*.iobj
-*.pch
-*.pdb
-*.ipdb
-*.pgc
-*.pgd
-*.rsp
-*.sbr
-*.tlb
-*.tli
-*.tlh
-*.tmp
-*.tmp_proj
-*_wpftmp.csproj
-*.log
-*.tlog
-*.vspscc
-*.vssscc
-.builds
-*.pidb
-*.svclog
-*.scc
-
-# Chutzpah Test files
-_Chutzpah*
-
-# Visual C++ cache files
-ipch/
-*.aps
-*.ncb
-*.opendb
-*.opensdf
-*.sdf
-*.cachefile
-*.VC.db
-*.VC.VC.opendb
-
-# Visual Studio profiler
-*.psess
-*.vsp
-*.vspx
-*.sap
-
-# Visual Studio Trace Files
-*.e2e
-
-# TFS 2012 Local Workspace
-$tf/
-
-# Guidance Automation Toolkit
-*.gpState
-
-# ReSharper is a .NET coding add-in
-_ReSharper*/
-*.[Rr]e[Ss]harper
-*.DotSettings.user
-
-# TeamCity is a build add-in
-_TeamCity*
-
-# DotCover is a Code Coverage Tool
-*.dotCover
-
-# AxoCover is a Code Coverage Tool
-.axoCover/*
-!.axoCover/settings.json
-
-# Coverlet is a free, cross platform Code Coverage Tool
-coverage*.json
-coverage*.xml
-coverage*.info
-
-# Visual Studio code coverage results
-*.coverage
-*.coveragexml
-
-# NCrunch
-_NCrunch_*
-.*crunch*.local.xml
-nCrunchTemp_*
-
-# MightyMoose
-*.mm.*
-AutoTest.Net/
-
-# Web workbench (sass)
-.sass-cache/
-
-# Installshield output folder
-[Ee]xpress/
-
-# DocProject is a documentation generator add-in
-DocProject/buildhelp/
-DocProject/Help/*.HxT
-DocProject/Help/*.HxC
-DocProject/Help/*.hhc
-DocProject/Help/*.hhk
-DocProject/Help/*.hhp
-DocProject/Help/Html2
-DocProject/Help/html
-
-# Click-Once directory
-publish/
-
-# Publish Web Output
-*.[Pp]ublish.xml
-*.azurePubxml
-# Note: Comment the next line if you want to checkin your web deploy settings,
-# but database connection strings (with potential passwords) will be unencrypted
-*.pubxml
-*.publishproj
-
-# Microsoft Azure Web App publish settings. Comment the next line if you want to
-# checkin your Azure Web App publish settings, but sensitive information contained
-# in these scripts will be unencrypted
-PublishScripts/
-
-# NuGet Packages
-*.nupkg
-# NuGet Symbol Packages
-*.snupkg
-# The packages folder can be ignored because of Package Restore
-**/[Pp]ackages/*
-# except build/, which is used as an MSBuild target.
-!**/[Pp]ackages/build/
-# Uncomment if necessary however generally it will be regenerated when needed
-#!**/[Pp]ackages/repositories.config
-# NuGet v3's project.json files produces more ignorable files
-*.nuget.props
-*.nuget.targets
-
-# Microsoft Azure Build Output
-csx/
-*.build.csdef
-
-# Microsoft Azure Emulator
-ecf/
-rcf/
-
-# Windows Store app package directories and files
-AppPackages/
-BundleArtifacts/
-Package.StoreAssociation.xml
-_pkginfo.txt
-*.appx
-*.appxbundle
-*.appxupload
-
-# Visual Studio cache files
-# files ending in .cache can be ignored
-*.[Cc]ache
-# but keep track of directories ending in .cache
-!?*.[Cc]ache/
-
-# Others
-ClientBin/
-~$*
-*~
-*.dbmdl
-*.dbproj.schemaview
-*.jfm
-*.pfx
-*.publishsettings
-orleans.codegen.cs
-
-# Including strong name files can present a security risk
-# (https://github.com/github/gitignore/pull/2483#issue-259490424)
-#*.snk
-
-# Since there are multiple workflows, uncomment next line to ignore bower_components
-# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
-#bower_components/
-
-# RIA/Silverlight projects
-Generated_Code/
-
-# Backup & report files from converting an old project file
-# to a newer Visual Studio version. Backup files are not needed,
-# because we have git ;-)
-_UpgradeReport_Files/
-Backup*/
-UpgradeLog*.XML
-UpgradeLog*.htm
-ServiceFabricBackup/
-*.rptproj.bak
-
-# SQL Server files
-*.mdf
-*.ldf
-*.ndf
-
-# Business Intelligence projects
-*.rdl.data
-*.bim.layout
-*.bim_*.settings
-*.rptproj.rsuser
-*- [Bb]ackup.rdl
-*- [Bb]ackup ([0-9]).rdl
-*- [Bb]ackup ([0-9][0-9]).rdl
-
-# Microsoft Fakes
-FakesAssemblies/
-
-# GhostDoc plugin setting file
-*.GhostDoc.xml
-
-# Node.js Tools for Visual Studio
-.ntvs_analysis.dat
-node_modules/
-
-# Visual Studio 6 build log
-*.plg
-
-# Visual Studio 6 workspace options file
-*.opt
-
-# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
-*.vbw
-
-# Visual Studio 6 auto-generated project file (contains which files were open etc.)
-*.vbp
-
-# Visual Studio 6 workspace and project file (working project files containing files to include in project)
-*.dsw
-*.dsp
-
-# Visual Studio 6 technical files
-*.ncb
-*.aps
-
-# Visual Studio LightSwitch build output
-**/*.HTMLClient/GeneratedArtifacts
-**/*.DesktopClient/GeneratedArtifacts
-**/*.DesktopClient/ModelManifest.xml
-**/*.Server/GeneratedArtifacts
-**/*.Server/ModelManifest.xml
-_Pvt_Extensions
-
-# Paket dependency manager
-.paket/paket.exe
-paket-files/
-
-# FAKE - F# Make
-.fake/
-
-# CodeRush personal settings
-.cr/personal
-
-# Python Tools for Visual Studio (PTVS)
+.venv/
__pycache__/
-*.pyc
-
-# Cake - Uncomment if you are using it
-# tools/**
-# !tools/packages.config
-
-# Tabs Studio
-*.tss
-
-# Telerik's JustMock configuration file
-*.jmconfig
-
-# BizTalk build output
-*.btp.cs
-*.btm.cs
-*.odx.cs
-*.xsd.cs
-
-# OpenCover UI analysis results
-OpenCover/
-
-# Azure Stream Analytics local run output
-ASALocalRun/
-
-# MSBuild Binary and Structured Log
-*.binlog
-
-# NVidia Nsight GPU debugger configuration file
-*.nvuser
-
-# MFractors (Xamarin productivity tool) working folder
-.mfractor/
-
-# Local History for Visual Studio
-.localhistory/
-
-# Visual Studio History (VSHistory) files
-.vshistory/
-
-# BeatPulse healthcheck temp database
-healthchecksdb
-
-# Backup folder for Package Reference Convert tool in Visual Studio 2017
-MigrationBackup/
-
-# Ionide (cross platform F# VS Code tools) working folder
-.ionide/
-
-# Fody - auto-generated XML schema
-FodyWeavers.xsd
-
-# VS Code files for those working on multiple tools
-.vscode/*
-!.vscode/settings.json
-!.vscode/tasks.json
-!.vscode/launch.json
-!.vscode/extensions.json
-*.code-workspace
-
-# Local History for Visual Studio Code
-.history/
-
-# Windows Installer files from build outputs
-*.cab
-*.msi
-*.msix
-*.msm
-*.msp
-
-# JetBrains Rider
-*.sln.iml
-
-# ---> VisualStudioCode
-.vscode/*
-!.vscode/settings.json
-!.vscode/tasks.json
-!.vscode/launch.json
-!.vscode/extensions.json
-!.vscode/*.code-snippets
-
-# Local History for Visual Studio Code
-.history/
-
-# Built Visual Studio Code Extensions
-*.vsix
-
+*.py[cod]
+.pytest_cache/
+.coverage
+htmlcov/
+.env
+*.db
+artifacts/
+dist/
+build/
+*.egg-info/
\ No newline at end of file
diff --git a/Containerfile b/Containerfile
new file mode 100644
index 0000000..3de9289
--- /dev/null
+++ b/Containerfile
@@ -0,0 +1,26 @@
+FROM python:3.12-slim AS runtime
+
+ENV PYTHONDONTWRITEBYTECODE=1 \
+ PYTHONUNBUFFERED=1 \
+ PIP_NO_CACHE_DIR=1
+
+RUN groupadd --system knowledgegraph \
+ && useradd --system --gid knowledgegraph --create-home knowledgegraph
+
+WORKDIR /app
+
+COPY pyproject.toml README.md alembic.ini ./
+COPY migrations ./migrations
+COPY src ./src
+
+RUN python -m pip install --upgrade pip \
+ && python -m pip install .
+
+RUN mkdir -p /app/artifacts \
+ && chown -R knowledgegraph:knowledgegraph /app
+
+USER knowledgegraph
+
+EXPOSE 8080
+
+CMD ["uvicorn", "graphrag_platform.api:app", "--host", "0.0.0.0", "--port", "8080"]
diff --git a/README.md b/README.md
index 978e4d4..574e893 100644
--- a/README.md
+++ b/README.md
@@ -1,2 +1,642 @@
-# RAG-POC
+# KnowledgeGraph GraphRAG
+An internal, restartable GraphRAG pipeline that normalizes externally collected
+content, extracts and reconciles knowledge, builds a graph, indexes graph and source
+evidence with PostgreSQL/PgVector, retrieves graph-aware context, and generates
+source-cited answers.
+
+The collection process is outside this project. Existing **Web-Access MCP** and
+**Data-Store MCP** processes discover and persist source material. This project
+starts by validating and normalizing their versioned handoff.
+
+## Architecture
+
+```text
+Web-Access MCP + Data-Store MCP
+ |
+ | versioned JSONL or PostgreSQL handoff
+ v
+01 normalize and parse
+ v
+02 clean, deduplicate, and chunk
+ v
+03 define knowledge model
+ v
+04 extract entities and relationships
+ v
+05 reconcile aliases and duplicates
+ v
+06 materialize graph nodes and edges
+ v
+07 detect and summarize communities
+ v
+08 build PgVector HNSW retrieval index
+ v
+09 retrieve vector + graph + source context
+ v
+10 generate a cited answer
+```
+
+PostgreSQL is the canonical checkpoint store. Every completed build stage persists
+its output, so later stages can be rebuilt without repeating valid upstream work.
+PgVector stores embeddings and performs server-side HNSW cosine search. Indexed
+relational edge tables provide bounded graph expansion.
+
+The deployment consists of:
+
+- PostgreSQL 16 with PgVector
+- Alembic migration job
+- Internal FastAPI service
+- PostgreSQL-backed asynchronous worker
+- Existing BFF as the policy enforcement point
+
+The API is internal only and is not intended for direct customer access.
+
+## Repository layout
+
+| Path | Purpose |
+|---|---|
+| [`src/01_normalize_content.py`](src/01_normalize_content.py) through [`src/10_generate_answer.py`](src/10_generate_answer.py) | Complete numbered stage implementations |
+| [`src/graphrag_platform`](src/graphrag_platform) | Configuration, models, storage, orchestration, API, auth, worker, and provider clients |
+| [`migrations`](migrations) | Alembic database migrations |
+| [`tests`](tests) | Unit, API/worker, pipeline, migration, and real PgVector integration tests |
+| [`compose.yaml`](compose.yaml) | Reproducible PostgreSQL, migration, API, and worker environment |
+| [`Containerfile`](Containerfile) | Non-root application image |
+| [`.env.example`](.env.example) | Runtime configuration template |
+
+## Prerequisites
+
+- Python 3.11 or newer
+- Podman 5 or newer
+- `podman machine` running on Windows
+- An OpenAI-compatible chat and embeddings endpoint
+- External Web-Access MCP/Data-Store MCP collection process
+
+## Quick start with Compose
+
+Create the local environment:
+
+```powershell
+Copy-Item .env.example .env
+```
+
+Edit `.env` and set at minimum:
+
+```dotenv
+POSTGRES_PASSWORD=choose-a-local-password
+DATABASE_URL=postgresql+psycopg://knowledgegraph_app:choose-a-local-password@localhost:5432/knowledgegraph
+
+LLM_BASE_URL=https://api.openai.com/v1
+LLM_API_KEY=replace-with-provider-key
+LLM_CHAT_MODEL=gpt-4.1-mini
+LLM_EMBEDDING_MODEL=text-embedding-3-small
+LLM_EMBEDDING_DIMENSIONS=1536
+
+INTERNAL_JWT_SECRET=replace-with-at-least-32-random-bytes
+```
+
+`POSTGRES_PASSWORD` and the password in `DATABASE_URL` must match. Compose replaces
+the database host with `postgres` inside application containers.
+
+Start the complete environment:
+
+```powershell
+podman machine start
+podman compose up --detach --build
+```
+
+Compose:
+
+1. Starts `docker.io/pgvector/pgvector:pg16`.
+2. Waits for PostgreSQL readiness.
+3. Runs `alembic upgrade head`.
+4. Starts the API and worker.
+5. Waits for API dependency readiness.
+
+Verify it:
+
+```powershell
+podman compose ps
+Invoke-RestMethod http://127.0.0.1:8080/internal/knowledge-graph/health
+Invoke-RestMethod http://127.0.0.1:8080/internal/knowledge-graph/ready
+```
+
+View logs:
+
+```powershell
+podman compose logs --follow api worker
+```
+
+Stop services while retaining PostgreSQL data:
+
+```powershell
+podman compose down
+```
+
+Delete services and all local database data:
+
+```powershell
+podman compose down --volumes
+```
+
+Volume removal is destructive.
+
+## External source handoff
+
+### JSON Lines contract
+
+Each UTF-8 JSONL line must satisfy schema version `1.0`:
+
+```json
+{
+ "schema_version": "1.0",
+ "external_id": "stable-source-record-id",
+ "corpus": "product-knowledge",
+ "url": "https://docs.example.com/article",
+ "body": "
Article text",
+ "content_type": "text/html",
+ "status_code": 200,
+ "retrieved_at": "2026-10-13T18:00:00Z",
+ "metadata": {
+ "collector": "Web-Access",
+ "stored_by": "Data-Store"
+ }
+}
+```
+
+Required fields:
+
+- `schema_version`: currently `1.0`
+- `external_id`: stable and unique within a corpus
+- `url`
+- `body`
+- `retrieved_at`: ISO 8601 timestamp with offset
+
+Optional/defaulted fields:
+
+- `corpus`: when provided, it must match the selected runtime corpus
+- `content_type`: defaults to `text/html`
+- `status_code`: defaults to `200`
+- `metadata`: defaults to an empty object
+
+Unknown fields and unsupported schema versions are rejected. Invalid records include
+the file and line number in the error.
+
+The template is [`src/templates/mcp_documents.jsonl`](src/templates/mcp_documents.jsonl).
+A larger acceptance corpus is
+[`tests/fixtures/external_documents.jsonl`](tests/fixtures/external_documents.jsonl).
+
+### PostgreSQL source contract
+
+Set `SOURCE_DATABASE_URL` and `SOURCE_TABLE_NAME` to read the same contract from an
+external PostgreSQL table:
+
+```dotenv
+SOURCE_DATABASE_URL=postgresql+psycopg://source_reader:password@source-host:5432/source_database
+SOURCE_TABLE_NAME=external_documents
+```
+
+Compatible table:
+
+```sql
+CREATE TABLE external_documents (
+ external_id text PRIMARY KEY,
+ schema_version text NOT NULL DEFAULT '1.0',
+ corpus text,
+ url text NOT NULL,
+ body text NOT NULL,
+ content_type text NOT NULL DEFAULT 'text/html',
+ status_code integer NOT NULL DEFAULT 200,
+ retrieved_at timestamptz NOT NULL,
+ metadata jsonb NOT NULL DEFAULT '{}'::jsonb
+);
+```
+
+Additional source-table columns are ignored. The application account should receive
+read-only access to the source table.
+
+Provide either `--input-jsonl` or `SOURCE_DATABASE_URL`, not both.
+
+## Durable checkpoints
+
+Build stages persist these boundaries:
+
+```text
+raw_documents + documents
+ -> chunks
+ -> knowledge_models
+ -> extractions
+ -> resolved_entities + resolved_relationships
+ -> nodes + edges
+ -> communities
+ -> retrieval_items
+```
+
+`pipeline_runs` and `stage_runs` separately record:
+
+- queued/running/succeeded/failed state
+- selected stage range
+- attempt
+- configuration fingerprint
+- caller and correlation ID
+- stage inputs and outputs
+- timestamps and error details
+
+The worker requeues runs abandoned beyond `WORKER_STALE_RUN_SECONDS`. Set this value
+above the expected maximum duration of one stage.
+
+PostgreSQL advisory locks allow only one active build for a corpus. If another
+worker claims a job for that corpus, the job is returned to the queue instead of
+executing conflicting writes.
+
+DataFrames are not stage checkpoints. A Pandas or Polars DataFrame may be used by a
+future adapter for one bounded transformation, but every completed result must be
+persisted before the next stage starts.
+
+## Database migrations
+
+Runtime services verify the migrated schema and do not silently mutate it.
+
+Apply migrations:
+
+```powershell
+alembic upgrade head
+```
+
+Inspect the current revision:
+
+```powershell
+alembic current
+alembic history
+```
+
+Create a migration after changing SQLAlchemy models:
+
+```powershell
+alembic revision --autogenerate -m "describe schema change"
+```
+
+Review generated migrations before applying them. Compose runs upgrades through the
+one-shot `migrate` service before starting API and worker services.
+
+`AUTO_CREATE_SCHEMA=true` exists only for isolated tests. Do not enable it in shared
+dev, test, or production environments.
+
+## Run stages directly
+
+Install a local editable environment:
+
+```powershell
+py -3.12 -m venv .venv
+.\.venv\Scripts\Activate.ps1
+python -m pip install --upgrade pip
+python -m pip install -e ".[test]"
+```
+
+Run migrations before the first command:
+
+```powershell
+alembic upgrade head
+```
+
+### 1. Normalize external content
+
+```powershell
+python .\src\01_normalize_content.py `
+ --corpus product-knowledge `
+ --input-jsonl .\artifacts\mcp-documents.jsonl
+```
+
+This validates the external handoff, stores raw provenance, runs Trafilatura for
+HTML, and persists normalized document records.
+
+### 2. Clean, deduplicate, and chunk
+
+```powershell
+python .\src\02_clean_content.py --corpus product-knowledge
+```
+
+### 3. Define the knowledge model
+
+```powershell
+python .\src\03_define_knowledge_model.py `
+ --corpus product-knowledge `
+ --model-file .\src\templates\knowledge_model.json
+```
+
+### 4. Extract entities and relationships
+
+```powershell
+python .\src\04_extract_knowledge.py --corpus product-knowledge
+```
+
+LLM calls occur outside long database transactions. Each validated chunk result is
+committed independently.
+
+### 5. Reconcile knowledge
+
+```powershell
+python .\src\05_reconcile_knowledge.py --corpus product-knowledge
+```
+
+### 6. Build graph nodes and edges
+
+```powershell
+python .\src\06_build_knowledge_graph.py --corpus product-knowledge
+```
+
+### 7. Build communities
+
+```powershell
+python .\src\07_build_communities.py --corpus product-knowledge
+```
+
+### 8. Build the PgVector index
+
+```powershell
+python .\src\08_build_retrieval_index.py --corpus product-knowledge
+```
+
+This embeds chunks, nodes, edges, and community summaries, validates dimensions,
+stores vectors, and creates a dimension-specific HNSW cosine index.
+
+### 9. Retrieve GraphRAG context
+
+```powershell
+python .\src\09_query_graphrag.py `
+ --corpus product-knowledge `
+ --question "Who created Atlas?" `
+ --top-k 8
+```
+
+The result contains vector `context`, expanded `graph_context`, and traceable
+`sources`.
+
+### 10. Generate a cited answer
+
+```powershell
+python .\src\10_generate_answer.py `
+ --corpus product-knowledge `
+ --question "Who created Atlas?"
+```
+
+## Run a durable build
+
+Run stages 1 through 8 under one durable pipeline record:
+
+```powershell
+knowledgegraph run `
+ --corpus product-knowledge `
+ --from-stage 1 `
+ --through-stage 8 `
+ --input-jsonl .\artifacts\mcp-documents.jsonl `
+ --model-file .\src\templates\knowledge_model.json `
+ --requested-by local-developer
+```
+
+Use `--from-stage` to rebuild only downstream state. The `--reset` option deletes
+the selected stage and downstream derived corpus data before execution.
+
+## BFF authorization contract
+
+The existing BFF authenticates callers and evaluates the `KnowledgeGraph` feature.
+Independent permission bits are:
+
+```text
+Read = 1
+Add = 2
+Edit = 4
+Delete = 8
+Execute = 16
+```
+
+| Operation | Required mask |
+|---|---:|
+| Read job/source/retrieval information | `Read` (`1`) |
+| Submit a stage-1 source job | `Add | Execute` (`18`) |
+| Run downstream build stages | `Execute` (`16`) |
+| Query or answer | `Read | Execute` (`17`) |
+| Retry a failed job | `Execute` (`16`) |
+
+The BFF forwards a signed internal JWT. Required claims:
+
+```json
+{
+ "iss": "knowledgegraph-bff",
+ "aud": "knowledgegraph-service",
+ "sub": "internal-caller-id",
+ "iat": 1791930000,
+ "exp": 1791930600,
+ "jti": "correlation-or-request-id",
+ "features": {
+ "KnowledgeGraph": 31
+ }
+}
+```
+
+Dev/test uses HS256 and `INTERNAL_JWT_SECRET`. Production deployments can replace
+this verifier with the BFF's workload identity, mTLS, or asymmetric/JWKS contract.
+Never trust unsigned permission headers.
+
+## Internal API
+
+| Endpoint | Purpose |
+|---|---|
+| `GET /internal/knowledge-graph/health` | Process liveness |
+| `GET /internal/knowledge-graph/ready` | Database and PgVector readiness |
+| `POST /internal/knowledge-graph/jobs` | Queue stages 1–8 |
+| `GET /internal/knowledge-graph/jobs/{jobId}` | Read durable job/stage status |
+| `POST /internal/knowledge-graph/jobs/{jobId}/retry` | Requeue a failed job |
+| `POST /internal/knowledge-graph/query` | Synchronous bounded retrieval |
+| `POST /internal/knowledge-graph/answer` | Synchronous cited answer |
+
+Example job request:
+
+```json
+{
+ "corpus": "product-knowledge",
+ "from_stage": 1,
+ "through_stage": 8,
+ "input_jsonl": "/app/artifacts/mcp-documents.jsonl",
+ "model_file": "/app/src/templates/knowledge_model.json",
+ "reset": false
+}
+```
+
+Compose bind-mounts local [`artifacts`](artifacts) at `/app/artifacts`, allowing the
+worker to read JSONL handoffs placed there. In deployed environments, prefer the
+PostgreSQL source contract or a shared managed file volume.
+
+Long-running stages return `202 Accepted` with a job ID. Query and answer requests
+remain synchronous and bounded.
+
+## Retry and logging behavior
+
+Model-provider calls retry:
+
+- HTTP 429
+- HTTP 500, 502, 503, and 504
+- connection failures
+- timeouts
+
+Retries honor numeric `Retry-After`, otherwise use bounded exponential backoff with
+jitter. Authentication and validation errors are not retried.
+
+Logs are emitted as one JSON object per line and include event names, run IDs,
+corpus/stage context where available, attempts, status, and error details. Secrets
+are not logged.
+
+## Tests
+
+Portable tests:
+
+```powershell
+pytest --ignore=.\tests\integration\test_pgvector_postgres.py
+```
+
+Real PostgreSQL/PgVector migration and retrieval test:
+
+```powershell
+$env:TEST_POSTGRES_URL = "postgresql+psycopg://knowledgegraph_app:password@localhost:5432/knowledgegraph"
+pytest .\tests\integration\test_pgvector_postgres.py
+Remove-Item Env:TEST_POSTGRES_URL
+```
+
+The suite covers:
+
+- Versioned source validation
+- Nested metadata and quoted/comma-containing content
+- Configuration parsing
+- Chunk validation
+- Cross-chunk relationship reconciliation
+- Provider retry policy
+- Full ten-stage behavior
+- Signed BFF authorization and permission failures
+- Durable API job and worker execution
+- Alembic migration against PostgreSQL
+- PgVector extension and HNSW index creation
+- Server-side vector retrieval, graph expansion, and citations
+
+Opt-in vector load test:
+
+```powershell
+$env:TEST_POSTGRES_URL = "postgresql+psycopg://knowledgegraph_app:password@localhost:5432/knowledgegraph"
+$env:RUN_PERFORMANCE_TESTS = "1"
+$env:PERF_VECTOR_ITEM_COUNT = "10000"
+$env:PERF_MAX_QUERY_SECONDS = "2.0"
+pytest .\tests\performance\test_vector_query_load.py
+```
+
+Set the item count and latency threshold to the capacity target for the dev/test
+environment. Performance checks are excluded by default because machine resources
+vary.
+
+CI is defined in [`.github/workflows/ci.yml`](.github/workflows/ci.yml) and compiles,
+tests, validates Compose, exercises real PgVector, and builds the application image.
+
+## Persistent format policy
+
+- PostgreSQL is the canonical operational store.
+- JSON Lines is the preferred file handoff/checkpoint format.
+- CSV is allowed only for explicitly defined flat import/export views.
+
+CSV safely supports commas, quotes, and line breaks when written by a compliant CSV
+library. It remains unsuitable as the canonical graph format because aliases,
+metadata, source ID arrays, embeddings, and graph properties are nested and typed.
+Never construct CSV by joining strings with commas.
+
+## RAG database alternatives
+
+### PostgreSQL with PgVector
+
+Implemented default. Choose it for SQL transactions, colocated provenance and
+vectors, self-managed deployment, and tunable HNSW retrieval. Graph traversal uses
+indexed relational edges rather than a native property-graph language.
+
+Official reference: [PgVector](https://github.com/pgvector/pgvector).
+
+### Azure Cosmos DB for NoSQL
+
+Choose it for managed/serverless document storage, vector indexing, and first-party
+vector plus BM25 hybrid search. Cosmos DB's Gremlin API is separate from the NoSQL
+vector feature; use document adjacency or a separate graph store.
+
+References:
+
+- [Cosmos DB vector search](https://learn.microsoft.com/azure/cosmos-db/vector-search)
+- [Cosmos DB hybrid search](https://learn.microsoft.com/azure/cosmos-db/gen-ai/hybrid-search)
+- [Cosmos DB Gremlin API](https://learn.microsoft.com/azure/cosmos-db/gremlin/overview)
+
+### Neo4j
+
+Choose it when graph topology, paths, and multi-hop traversal are primary retrieval
+signals. Neo4j combines native property graphs, Cypher, vector indexes, and
+graph-aware hybrid retrieval. It requires a dedicated backend adapter; changing
+`DATABASE_URL` is not sufficient. Confirm version and edition requirements for the
+selected vector and unified-search features.
+
+References:
+
+- [Neo4j vector indexes](https://neo4j.com/docs/cypher-manual/current/indexes/search-indexes/vector-indexes/)
+- [Neo4j hybrid search](https://neo4j.com/developer/genai-ecosystem/hybrid-search/)
+
+Other credible options:
+
+- [Amazon Neptune Analytics](https://docs.aws.amazon.com/neptune-analytics/latest/userguide/what-is-neptune-analytics.html)
+ for AWS-managed graph/vector analytics
+- [Azure AI Search](https://learn.microsoft.com/azure/search/hybrid-search-overview)
+ as a secondary managed hybrid index without a native graph model
+
+## Operational guidance
+
+- Keep the API and worker on internal networks accessible only to the BFF and
+ operators.
+- Replace all dev secrets before shared deployment.
+- Use a separate PostgreSQL role for migrations and a least-privilege runtime role.
+- Back up PostgreSQL before destructive resets or migrations.
+- Monitor queued/running/failed run counts, stage duration, LLM retries, database
+ pool usage, vector latency, and retrieval result counts.
+- Set `WORKER_STALE_RUN_SECONDS` above the longest expected single stage.
+- Rebuild stage 8 after changing embedding model or dimensions.
+- Maintain a golden question/citation set to detect retrieval-quality regressions.
+
+## Troubleshooting
+
+### Database migrations are required
+
+```powershell
+alembic current
+alembic upgrade head
+```
+
+### `LLM_API_KEY is required`
+
+Set `LLM_API_KEY` in `.env` and restart API/worker containers.
+
+### Embedding dimension mismatch
+
+Set `LLM_EMBEDDING_DIMENSIONS` to the exact provider output dimension, then reset
+and rerun stage 8.
+
+### Source handoff rejected
+
+Validate every JSONL line against schema version `1.0`. Confirm `corpus`,
+`external_id`, URL, body, and `retrieved_at`.
+
+### Job remains queued
+
+```powershell
+podman compose ps worker
+podman compose logs --tail 100 worker
+```
+
+### PostgreSQL/PgVector is not ready
+
+```powershell
+podman compose ps postgres
+podman compose logs --tail 100 postgres migrate
+```
+
+### Port conflict
+
+Change `POSTGRES_PORT` or `API_PORT` in `.env`, then recreate the Compose stack.
diff --git a/alembic.ini b/alembic.ini
new file mode 100644
index 0000000..1f3d3e9
--- /dev/null
+++ b/alembic.ini
@@ -0,0 +1,38 @@
+[alembic]
+script_location = migrations
+prepend_sys_path = .
+path_separator = os
+
+[loggers]
+keys = root,sqlalchemy,alembic
+
+[handlers]
+keys = console
+
+[formatters]
+keys = generic
+
+[logger_root]
+level = WARN
+handlers = console
+qualname =
+
+[logger_sqlalchemy]
+level = WARN
+handlers =
+qualname = sqlalchemy.engine
+
+[logger_alembic]
+level = INFO
+handlers =
+qualname = alembic
+
+[handler_console]
+class = StreamHandler
+args = (sys.stderr,)
+level = NOTSET
+formatter = generic
+
+[formatter_generic]
+format = %(levelname)-5.5s [%(name)s] %(message)s
+datefmt = %H:%M:%S
diff --git a/compose.yaml b/compose.yaml
new file mode 100644
index 0000000..5ffe829
--- /dev/null
+++ b/compose.yaml
@@ -0,0 +1,99 @@
+name: knowledgegraph
+
+services:
+ postgres:
+ image: docker.io/pgvector/pgvector:pg16
+ environment:
+ POSTGRES_USER: knowledgegraph_app
+ POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-knowledgegraph-dev}
+ POSTGRES_DB: knowledgegraph
+ ports:
+ - "127.0.0.1:${POSTGRES_PORT:-5432}:5432"
+ volumes:
+ - knowledgegraph-postgres-data:/var/lib/postgresql/data
+ healthcheck:
+ test:
+ [
+ "CMD-SHELL",
+ "pg_isready -U knowledgegraph_app -d knowledgegraph",
+ ]
+ interval: 5s
+ timeout: 5s
+ retries: 20
+ restart: unless-stopped
+
+ migrate:
+ build:
+ context: .
+ dockerfile: Containerfile
+ command: ["alembic", "upgrade", "head"]
+ environment: &application-environment
+ DATABASE_URL: postgresql+psycopg://knowledgegraph_app:${POSTGRES_PASSWORD:-knowledgegraph-dev}@postgres:5432/knowledgegraph
+ SOURCE_DATABASE_URL: ${SOURCE_DATABASE_URL:-}
+ SOURCE_TABLE_NAME: ${SOURCE_TABLE_NAME:-external_documents}
+ AUTO_CREATE_SCHEMA: "false"
+ CORPUS_NAME: ${CORPUS_NAME:-default-corpus}
+ LLM_BASE_URL: ${LLM_BASE_URL:-https://api.openai.com/v1}
+ LLM_API_KEY: ${LLM_API_KEY:-}
+ LLM_CHAT_MODEL: ${LLM_CHAT_MODEL:-gpt-4.1-mini}
+ LLM_EMBEDDING_MODEL: ${LLM_EMBEDDING_MODEL:-text-embedding-3-small}
+ LLM_EMBEDDING_DIMENSIONS: ${LLM_EMBEDDING_DIMENSIONS:-1536}
+ INTERNAL_JWT_SECRET: ${INTERNAL_JWT_SECRET:-knowledgegraph-dev-secret-change-this-32}
+ INTERNAL_JWT_ISSUER: ${INTERNAL_JWT_ISSUER:-knowledgegraph-bff}
+ INTERNAL_JWT_AUDIENCE: ${INTERNAL_JWT_AUDIENCE:-knowledgegraph-service}
+ INTERNAL_FEATURE_NAME: ${INTERNAL_FEATURE_NAME:-KnowledgeGraph}
+ LOG_LEVEL: ${LOG_LEVEL:-INFO}
+ depends_on:
+ postgres:
+ condition: service_healthy
+ restart: "no"
+
+ api:
+ build:
+ context: .
+ dockerfile: Containerfile
+ command:
+ [
+ "uvicorn",
+ "graphrag_platform.api:app",
+ "--host",
+ "0.0.0.0",
+ "--port",
+ "8080",
+ ]
+ environment: *application-environment
+ ports:
+ - "127.0.0.1:${API_PORT:-8080}:8080"
+ volumes:
+ - ./artifacts:/app/artifacts
+ depends_on:
+ migrate:
+ condition: service_completed_successfully
+ healthcheck:
+ test:
+ [
+ "CMD",
+ "python",
+ "-c",
+ "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/internal/knowledge-graph/ready', timeout=5)",
+ ]
+ interval: 10s
+ timeout: 5s
+ retries: 12
+ restart: unless-stopped
+
+ worker:
+ build:
+ context: .
+ dockerfile: Containerfile
+ command: ["knowledgegraph-worker"]
+ environment: *application-environment
+ volumes:
+ - ./artifacts:/app/artifacts
+ depends_on:
+ migrate:
+ condition: service_completed_successfully
+ restart: unless-stopped
+
+volumes:
+ knowledgegraph-postgres-data:
diff --git a/migrations/env.py b/migrations/env.py
new file mode 100644
index 0000000..8cc2451
--- /dev/null
+++ b/migrations/env.py
@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from logging.config import fileConfig
+
+from alembic import context
+from sqlalchemy import engine_from_config, pool
+
+from graphrag_platform.config import Settings
+from graphrag_platform.models import Base
+
+config = context.config
+if config.config_file_name:
+ fileConfig(config.config_file_name)
+
+config.set_main_option("sqlalchemy.url", Settings().database_url)
+target_metadata = Base.metadata
+
+
+def run_migrations_offline() -> None:
+ context.configure(
+ url=config.get_main_option("sqlalchemy.url"),
+ target_metadata=target_metadata,
+ literal_binds=True,
+ dialect_opts={"paramstyle": "named"},
+ compare_type=True,
+ )
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+def run_migrations_online() -> None:
+ connectable = engine_from_config(
+ config.get_section(config.config_ini_section, {}),
+ prefix="sqlalchemy.",
+ poolclass=pool.NullPool,
+ )
+ with connectable.connect() as connection:
+ context.configure(
+ connection=connection,
+ target_metadata=target_metadata,
+ compare_type=True,
+ )
+ with context.begin_transaction():
+ context.run_migrations()
+
+
+if context.is_offline_mode():
+ run_migrations_offline()
+else:
+ run_migrations_online()
diff --git a/migrations/script.py.mako b/migrations/script.py.mako
new file mode 100644
index 0000000..590f5b3
--- /dev/null
+++ b/migrations/script.py.mako
@@ -0,0 +1,24 @@
+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+"""
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+revision: str = ${repr(up_revision)}
+down_revision: Union[str, None] = ${repr(down_revision)}
+branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
+depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
+
+
+def upgrade() -> None:
+ ${upgrades if upgrades else "pass"}
+
+
+def downgrade() -> None:
+ ${downgrades if downgrades else "pass"}
diff --git a/migrations/versions/20261013_0001_initial.py b/migrations/versions/20261013_0001_initial.py
new file mode 100644
index 0000000..61eafbb
--- /dev/null
+++ b/migrations/versions/20261013_0001_initial.py
@@ -0,0 +1,25 @@
+"""Create the initial KnowledgeGraph schema.
+
+Revision ID: 20261013_0001
+Revises:
+Create Date: 2026-10-13
+"""
+
+from alembic import op
+
+from graphrag_platform.models import Base
+
+revision = "20261013_0001"
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade() -> None:
+ connection = op.get_bind()
+ connection.exec_driver_sql("CREATE EXTENSION IF NOT EXISTS vector")
+ Base.metadata.create_all(bind=connection)
+
+
+def downgrade() -> None:
+ Base.metadata.drop_all(bind=op.get_bind())
diff --git a/pyproject.toml b/pyproject.toml
new file mode 100644
index 0000000..15147dd
--- /dev/null
+++ b/pyproject.toml
@@ -0,0 +1,60 @@
+[build-system]
+requires = ["hatchling>=1.25"]
+build-backend = "hatchling.build"
+
+[project]
+name = "knowledgegraph-graphrag"
+version = "0.1.0"
+description = "Production-oriented KnowledgeGraph ingestion, indexing, retrieval, and answer pipeline"
+readme = "README.md"
+requires-python = ">=3.11"
+dependencies = [
+ "alembic>=1.13,<2",
+ "fastapi>=0.115,<1",
+ "httpx>=0.27,<1",
+ "networkx>=3.3,<4",
+ "pgvector>=0.3,<1",
+ "PyJWT>=2.9,<3",
+ "pydantic>=2.8,<3",
+ "pydantic-settings>=2.4,<3",
+ "psycopg[binary]>=3.2,<4",
+ "sqlalchemy>=2.0,<3",
+ "trafilatura>=1.12,<3",
+ "uvicorn[standard]>=0.32,<1",
+]
+
+[project.optional-dependencies]
+test = [
+ "pytest>=8.3,<9",
+ "pytest-cov>=5,<7",
+]
+
+[project.scripts]
+knowledgegraph = "graphrag_platform.cli:main_all"
+knowledgegraph-worker = "graphrag_platform.worker:main"
+
+[tool.hatch.build.targets.wheel]
+packages = [
+ "src/graphrag_platform",
+ "src/01_normalize_content.py",
+ "src/02_clean_content.py",
+ "src/03_define_knowledge_model.py",
+ "src/04_extract_knowledge.py",
+ "src/05_reconcile_knowledge.py",
+ "src/06_build_knowledge_graph.py",
+ "src/07_build_communities.py",
+ "src/08_build_retrieval_index.py",
+ "src/09_query_graphrag.py",
+ "src/10_generate_answer.py",
+]
+
+[tool.pytest.ini_options]
+addopts = "-ra"
+testpaths = ["tests"]
+markers = [
+ "performance: opt-in performance and load checks",
+]
+
+[tool.coverage.run]
+source = ["graphrag_platform"]
+branch = true
\ No newline at end of file
diff --git a/src/01_normalize_content.py b/src/01_normalize_content.py
new file mode 100644
index 0000000..440c7ea
--- /dev/null
+++ b/src/01_normalize_content.py
@@ -0,0 +1,128 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+
+import trafilatura
+
+from graphrag_platform.cli import stage_main
+from graphrag_platform.models import Document, RawDocument
+from graphrag_platform.runtime import StageContext
+from graphrag_platform.sources import SourceRecord, source_reader
+
+
+def content_hash(text: str) -> str:
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def extract_document(record: SourceRecord) -> dict[str, object]:
+ if "html" not in record.content_type.casefold():
+ text = record.body.strip()
+ if not text:
+ raise ValueError(f"source content is empty for {record.url}")
+ return {
+ "title": record.metadata.get("title"),
+ "author": record.metadata.get("author"),
+ "date": record.metadata.get("published_at"),
+ "hostname": None,
+ "sitename": None,
+ "text": text,
+ }
+ extracted = trafilatura.extract(
+ record.body,
+ url=str(record.url),
+ output_format="json",
+ with_metadata=True,
+ include_comments=False,
+ include_tables=True,
+ favor_precision=True,
+ )
+ if not extracted:
+ raise ValueError(f"could not extract article content from {record.url}")
+ payload = json.loads(extracted)
+ if not str(payload.get("text") or "").strip():
+ raise ValueError(f"extracted content is empty for {record.url}")
+ return payload
+
+
+def run(
+ context: StageContext,
+ *,
+ input_jsonl: Path | None = None,
+) -> dict[str, int]:
+ context.initialize()
+ reader = source_reader(
+ input_jsonl=input_jsonl,
+ source_database_url=context.settings.source_database_url,
+ source_table_name=context.settings.source_table_name,
+ )
+ normalized = 0
+ with context.storage.session() as session:
+ for record in reader.records():
+ if record.corpus and record.corpus != context.settings.corpus_name:
+ raise ValueError(
+ f"record {record.external_id} targets corpus {record.corpus}, "
+ f"not {context.settings.corpus_name}"
+ )
+ payload = extract_document(record)
+ text = str(payload["text"]).strip()
+ metadata = dict(record.metadata)
+ metadata.update(
+ {
+ "external_id": record.external_id,
+ "source_schema_version": record.schema_version,
+ "retrieved_at": record.retrieved_at.isoformat(),
+ "hostname": payload.get("hostname"),
+ "sitename": payload.get("sitename"),
+ }
+ )
+ raw = context.storage.upsert(
+ session,
+ RawDocument,
+ {
+ "corpus": context.settings.corpus_name,
+ "external_id": record.external_id,
+ },
+ {
+ "source_schema_version": record.schema_version,
+ "url": str(record.url),
+ "status_code": record.status_code,
+ "content_type": record.content_type,
+ "body": record.body,
+ "metadata_json": record.metadata,
+ "retrieved_at": record.retrieved_at,
+ },
+ )
+ context.storage.upsert(
+ session,
+ Document,
+ {"raw_document_id": raw.id},
+ {
+ "corpus": context.settings.corpus_name,
+ "url": str(record.url),
+ "title": payload.get("title"),
+ "author": payload.get("author"),
+ "published_at": payload.get("date"),
+ "text": text,
+ "content_hash": content_hash(text),
+ "metadata_json": metadata,
+ "is_duplicate": False,
+ "canonical_document_id": None,
+ },
+ )
+ normalized += 1
+ if normalized == 0:
+ raise ValueError("source handoff contained no records")
+ return {"normalized": normalized}
+
+
+def run_from_args(
+ context: StageContext, args: argparse.Namespace
+) -> dict[str, int]:
+ return run(context, input_jsonl=args.input_jsonl)
+
+
+if __name__ == "__main__":
+ stage_main(1, run_from_args)
diff --git a/src/02_clean_content.py b/src/02_clean_content.py
new file mode 100644
index 0000000..cd3bb3f
--- /dev/null
+++ b/src/02_clean_content.py
@@ -0,0 +1,103 @@
+from __future__ import annotations
+
+import argparse
+import hashlib
+import re
+
+from sqlalchemy import delete, select
+
+from graphrag_platform.cli import stage_main
+from graphrag_platform.models import Chunk, Document
+from graphrag_platform.runtime import StageContext
+
+_SPACE_RE = re.compile(r"[ \t]+")
+_PARAGRAPH_RE = re.compile(r"\n{3,}")
+
+
+def content_hash(text: str) -> str:
+ return hashlib.sha256(text.encode("utf-8")).hexdigest()
+
+
+def clean_text(text: str) -> str:
+ lines = [_SPACE_RE.sub(" ", line).strip() for line in text.splitlines()]
+ return _PARAGRAPH_RE.sub("\n\n", "\n".join(lines)).strip()
+
+
+def chunk_text(text: str, *, size: int, overlap: int) -> list[str]:
+ if overlap >= size:
+ raise ValueError("CHUNK_OVERLAP must be smaller than CHUNK_SIZE")
+ chunks: list[str] = []
+ start = 0
+ while start < len(text):
+ end = min(start + size, len(text))
+ if end < len(text):
+ break_at = text.rfind(" ", start + size // 2, end)
+ if break_at > start:
+ end = break_at
+ chunk = text[start:end].strip()
+ if chunk:
+ chunks.append(chunk)
+ if end >= len(text):
+ break
+ start = end - overlap
+ return chunks
+
+
+def run(context: StageContext) -> dict[str, int]:
+ context.initialize()
+ chunk_count = 0
+ duplicate_count = 0
+ with context.storage.session() as session:
+ documents = session.scalars(
+ select(Document)
+ .where(Document.corpus == context.settings.corpus_name)
+ .order_by(Document.id)
+ ).all()
+ if not documents:
+ raise ValueError("stage 2 requires normalized documents")
+ session.execute(
+ delete(Chunk).where(Chunk.corpus == context.settings.corpus_name)
+ )
+ hashes: dict[str, int] = {}
+ for document in documents:
+ text = clean_text(document.text)
+ digest = content_hash(text)
+ document.text = text
+ document.content_hash = digest
+ if digest in hashes:
+ document.is_duplicate = True
+ document.canonical_document_id = hashes[digest]
+ duplicate_count += 1
+ continue
+ document.is_duplicate = False
+ document.canonical_document_id = None
+ hashes[digest] = document.id
+ for ordinal, text_chunk in enumerate(
+ chunk_text(
+ text,
+ size=context.settings.chunk_size,
+ overlap=context.settings.chunk_overlap,
+ )
+ ):
+ session.add(
+ Chunk(
+ document_id=document.id,
+ corpus=context.settings.corpus_name,
+ ordinal=ordinal,
+ text=text_chunk,
+ content_hash=content_hash(text_chunk),
+ )
+ )
+ chunk_count += 1
+ return {"chunks": chunk_count, "duplicates": duplicate_count}
+
+
+def run_from_args(
+ context: StageContext, args: argparse.Namespace
+) -> dict[str, int]:
+ del args
+ return run(context)
+
+
+if __name__ == "__main__":
+ stage_main(2, run_from_args)
diff --git a/src/03_define_knowledge_model.py b/src/03_define_knowledge_model.py
new file mode 100644
index 0000000..2b8c25e
--- /dev/null
+++ b/src/03_define_knowledge_model.py
@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+from typing import Any
+
+from graphrag_platform.cli import stage_main
+from graphrag_platform.models import KnowledgeModel
+from graphrag_platform.runtime import StageContext
+
+
+def validate_model_definition(definition: dict[str, Any]) -> None:
+ for key in ("entity_types", "relationship_types"):
+ value = definition.get(key)
+ if not isinstance(value, list) or not value or not all(
+ isinstance(item, str) and item.strip() for item in value
+ ):
+ raise ValueError(f"{key} must be a non-empty array of strings")
+ for key in ("node_template", "edge_template"):
+ if not isinstance(definition.get(key), dict):
+ raise ValueError(f"{key} must be a JSON object")
+
+
+def run(
+ context: StageContext, *, model_file: Path | None = None
+) -> dict[str, int]:
+ context.initialize()
+ definition = context.settings.model_definition()
+ if model_file is not None:
+ loaded = json.loads(model_file.read_text(encoding="utf-8"))
+ if not isinstance(loaded, dict):
+ raise ValueError("knowledge model file must contain a JSON object")
+ definition.update(loaded)
+ validate_model_definition(definition)
+ with context.storage.session() as session:
+ context.storage.upsert(
+ session,
+ KnowledgeModel,
+ {"corpus": context.settings.corpus_name},
+ {"definition_json": definition},
+ )
+ return {
+ "entity_types": len(definition["entity_types"]),
+ "relationship_types": len(definition["relationship_types"]),
+ }
+
+
+def run_from_args(
+ context: StageContext, args: argparse.Namespace
+) -> dict[str, int]:
+ return run(context, model_file=args.model_file)
+
+
+if __name__ == "__main__":
+ stage_main(3, run_from_args)
diff --git a/src/04_extract_knowledge.py b/src/04_extract_knowledge.py
new file mode 100644
index 0000000..c98acb6
--- /dev/null
+++ b/src/04_extract_knowledge.py
@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import argparse
+import json
+from typing import Any
+
+from sqlalchemy import select
+
+from graphrag_platform.cli import stage_main
+from graphrag_platform.models import Chunk, Extraction, KnowledgeModel
+from graphrag_platform.runtime import StageContext
+
+
+def validate_extraction(
+ payload: dict[str, Any], definition: dict[str, Any]
+) -> None:
+ entities = payload.get("entities")
+ relationships = payload.get("relationships")
+ if not isinstance(entities, list) or not isinstance(relationships, list):
+ raise ValueError("extraction requires entities and relationships arrays")
+ entity_types = set(definition["entity_types"])
+ relationship_types = set(definition["relationship_types"])
+ for entity in entities:
+ if not isinstance(entity, dict):
+ raise ValueError("each entity must be an object")
+ if not str(entity.get("canonical_name", "")).strip():
+ raise ValueError("entity canonical_name must not be empty")
+ if entity.get("entity_type") not in entity_types:
+ raise ValueError(f"unsupported entity type: {entity.get('entity_type')}")
+ if not isinstance(entity.get("aliases", []), list):
+ raise ValueError("entity aliases must be an array")
+ for relationship in relationships:
+ if not isinstance(relationship, dict):
+ raise ValueError("each relationship must be an object")
+ if not str(relationship.get("source", "")).strip() or not str(
+ relationship.get("target", "")
+ ).strip():
+ raise ValueError("relationship source and target must not be empty")
+ if relationship.get("relationship_type") not in relationship_types:
+ raise ValueError(
+ "unsupported relationship type: "
+ f"{relationship.get('relationship_type')}"
+ )
+
+
+def run(context: StageContext) -> dict[str, int]:
+ context.initialize()
+ with context.storage.session() as session:
+ model = session.get(KnowledgeModel, context.settings.corpus_name)
+ chunks = session.scalars(
+ select(Chunk).where(Chunk.corpus == context.settings.corpus_name)
+ ).all()
+ if model is None or not chunks:
+ raise ValueError("stage 4 requires a knowledge model and cleaned chunks")
+ definition = dict(model.definition_json)
+ chunk_inputs = [(chunk.id, chunk.text) for chunk in chunks]
+
+ system = (
+ "You extract a source-grounded knowledge graph. Return one JSON object "
+ 'with arrays named "entities" and "relationships". Never infer unsupported facts.'
+ )
+ extracted_count = 0
+ for chunk_id, chunk_text in chunk_inputs:
+ prompt = json.dumps(
+ {
+ "knowledge_model": definition,
+ "source_chunk_id": chunk_id,
+ "text": chunk_text,
+ },
+ ensure_ascii=False,
+ )
+ payload = context.llm.complete_json(system, prompt)
+ validate_extraction(payload, definition)
+ with context.storage.session() as session:
+ context.storage.upsert(
+ session,
+ Extraction,
+ {"chunk_id": chunk_id},
+ {
+ "corpus": context.settings.corpus_name,
+ "payload_json": payload,
+ "model_name": context.llm.chat_model,
+ },
+ )
+ extracted_count += 1
+ return {"extracted_chunks": extracted_count}
+
+
+def run_from_args(
+ context: StageContext, args: argparse.Namespace
+) -> dict[str, int]:
+ del args
+ return run(context)
+
+
+if __name__ == "__main__":
+ stage_main(4, run_from_args)
diff --git a/src/05_reconcile_knowledge.py b/src/05_reconcile_knowledge.py
new file mode 100644
index 0000000..2dba4c9
--- /dev/null
+++ b/src/05_reconcile_knowledge.py
@@ -0,0 +1,137 @@
+from __future__ import annotations
+
+import argparse
+import re
+from typing import Any
+
+from sqlalchemy import delete, select
+
+from graphrag_platform.cli import stage_main
+from graphrag_platform.models import (
+ Extraction,
+ ResolvedEntity,
+ ResolvedRelationship,
+)
+from graphrag_platform.runtime import StageContext
+
+_KEY_RE = re.compile(r"[^\w]+", re.UNICODE)
+
+
+def canonical_key(name: str) -> str:
+ return _KEY_RE.sub(" ", name.casefold()).strip()
+
+
+def merge_text(current: str, incoming: str) -> str:
+ current = current.strip()
+ incoming = incoming.strip()
+ if not incoming or incoming.casefold() in current.casefold():
+ return current
+ if not current:
+ return incoming
+ return f"{current}\n{incoming}"
+
+
+def run(context: StageContext) -> dict[str, int]:
+ context.initialize()
+ entity_count = 0
+ relationship_count = 0
+ with context.storage.session() as session:
+ extractions = session.scalars(
+ select(Extraction).where(
+ Extraction.corpus == context.settings.corpus_name
+ )
+ ).all()
+ if not extractions:
+ raise ValueError("stage 5 requires extracted knowledge")
+ session.execute(
+ delete(ResolvedRelationship).where(
+ ResolvedRelationship.corpus == context.settings.corpus_name
+ )
+ )
+ session.execute(
+ delete(ResolvedEntity).where(
+ ResolvedEntity.corpus == context.settings.corpus_name
+ )
+ )
+ entity_rows: dict[str, dict[str, Any]] = {}
+ relation_rows: dict[tuple[str, str, str], dict[str, Any]] = {}
+ pending_relationships: list[tuple[int, dict[str, Any]]] = []
+ for extraction in extractions:
+ payload = extraction.payload_json
+ for entity in payload["entities"]:
+ name = entity["canonical_name"].strip()
+ key = canonical_key(name)
+ row = entity_rows.get(key)
+ if row is None:
+ row = {
+ "canonical_name": name,
+ "entity_type": entity["entity_type"],
+ "description": "",
+ "aliases": set(),
+ "chunk_ids": set(),
+ }
+ entity_rows[key] = row
+ elif row["entity_type"] != entity["entity_type"]:
+ raise ValueError(
+ f"conflicting entity types for {name}: "
+ f"{row['entity_type']} and {entity['entity_type']}"
+ )
+ row["description"] = merge_text(
+ row["description"], entity.get("description", "")
+ )
+ row["aliases"].update(entity.get("aliases", []))
+ row["aliases"].add(name)
+ row["chunk_ids"].add(extraction.chunk_id)
+ for relationship in payload["relationships"]:
+ pending_relationships.append((extraction.chunk_id, relationship))
+ for chunk_id, relationship in pending_relationships:
+ source_key = canonical_key(relationship["source"])
+ target_key = canonical_key(relationship["target"])
+ if source_key not in entity_rows or target_key not in entity_rows:
+ continue
+ relation_type = relationship["relationship_type"]
+ identity = (source_key, target_key, relation_type)
+ row = relation_rows.setdefault(
+ identity, {"description": "", "chunk_ids": set()}
+ )
+ row["description"] = merge_text(
+ row["description"], relationship.get("description", "")
+ )
+ row["chunk_ids"].add(chunk_id)
+ for key, row in entity_rows.items():
+ session.add(
+ ResolvedEntity(
+ corpus=context.settings.corpus_name,
+ canonical_key=key,
+ canonical_name=row["canonical_name"],
+ entity_type=row["entity_type"],
+ description=row["description"],
+ aliases_json=sorted(row["aliases"], key=str.casefold),
+ source_chunk_ids_json=sorted(row["chunk_ids"]),
+ )
+ )
+ entity_count += 1
+ for (source_key, target_key, relation_type), row in relation_rows.items():
+ session.add(
+ ResolvedRelationship(
+ corpus=context.settings.corpus_name,
+ source_key=source_key,
+ target_key=target_key,
+ relationship_type=relation_type,
+ description=row["description"],
+ source_chunk_ids_json=sorted(row["chunk_ids"]),
+ )
+ )
+ relationship_count += 1
+ return {"entities": entity_count, "relationships": relationship_count}
+
+
+def run_from_args(
+ context: StageContext, args: argparse.Namespace
+) -> dict[str, int]:
+ del args
+ return run(context)
+
+
+if __name__ == "__main__":
+ stage_main(5, run_from_args)
diff --git a/src/06_build_knowledge_graph.py b/src/06_build_knowledge_graph.py
new file mode 100644
index 0000000..a760cbc
--- /dev/null
+++ b/src/06_build_knowledge_graph.py
@@ -0,0 +1,80 @@
+from __future__ import annotations
+
+import argparse
+
+from sqlalchemy import delete, select
+
+from graphrag_platform.cli import stage_main
+from graphrag_platform.models import (
+ Edge,
+ Node,
+ ResolvedEntity,
+ ResolvedRelationship,
+)
+from graphrag_platform.runtime import StageContext
+
+
+def run(context: StageContext) -> dict[str, int]:
+ context.initialize()
+ with context.storage.session() as session:
+ entities = session.scalars(
+ select(ResolvedEntity).where(
+ ResolvedEntity.corpus == context.settings.corpus_name
+ )
+ ).all()
+ relationships = session.scalars(
+ select(ResolvedRelationship).where(
+ ResolvedRelationship.corpus == context.settings.corpus_name
+ )
+ ).all()
+ if not entities:
+ raise ValueError("stage 6 requires reconciled entities")
+ session.execute(
+ delete(Edge).where(Edge.corpus == context.settings.corpus_name)
+ )
+ session.execute(
+ delete(Node).where(Node.corpus == context.settings.corpus_name)
+ )
+ node_ids: dict[str, int] = {}
+ for entity in entities:
+ node = Node(
+ corpus=entity.corpus,
+ canonical_key=entity.canonical_key,
+ canonical_name=entity.canonical_name,
+ entity_type=entity.entity_type,
+ description=entity.description,
+ aliases_json=entity.aliases_json,
+ source_chunk_ids_json=entity.source_chunk_ids_json,
+ )
+ session.add(node)
+ session.flush()
+ node_ids[entity.canonical_key] = node.id
+ edge_count = 0
+ for relationship in relationships:
+ source_id = node_ids.get(relationship.source_key)
+ target_id = node_ids.get(relationship.target_key)
+ if source_id is None or target_id is None:
+ continue
+ session.add(
+ Edge(
+ corpus=relationship.corpus,
+ source_node_id=source_id,
+ target_node_id=target_id,
+ relationship_type=relationship.relationship_type,
+ description=relationship.description,
+ source_chunk_ids_json=relationship.source_chunk_ids_json,
+ )
+ )
+ edge_count += 1
+ return {"nodes": len(node_ids), "edges": edge_count}
+
+
+def run_from_args(
+ context: StageContext, args: argparse.Namespace
+) -> dict[str, int]:
+ del args
+ return run(context)
+
+
+if __name__ == "__main__":
+ stage_main(6, run_from_args)
diff --git a/src/07_build_communities.py b/src/07_build_communities.py
new file mode 100644
index 0000000..89ff15f
--- /dev/null
+++ b/src/07_build_communities.py
@@ -0,0 +1,97 @@
+from __future__ import annotations
+
+import argparse
+import json
+
+import networkx as nx
+from sqlalchemy import delete, select
+
+from graphrag_platform.cli import stage_main
+from graphrag_platform.models import Community, Edge, Node
+from graphrag_platform.runtime import StageContext
+
+
+def run(context: StageContext) -> dict[str, int]:
+ context.initialize()
+ with context.storage.session() as session:
+ nodes = session.scalars(
+ select(Node).where(Node.corpus == context.settings.corpus_name)
+ ).all()
+ edges = session.scalars(
+ select(Edge).where(Edge.corpus == context.settings.corpus_name)
+ ).all()
+ if not nodes:
+ raise ValueError("stage 7 requires graph nodes")
+ graph = nx.Graph()
+ graph.add_nodes_from(node.id for node in nodes)
+ graph.add_edges_from(
+ (edge.source_node_id, edge.target_node_id) for edge in edges
+ )
+ if graph.number_of_edges():
+ groups = list(
+ nx.community.greedy_modularity_communities(
+ graph, resolution=context.settings.community_resolution
+ )
+ )
+ else:
+ groups = [{node.id} for node in nodes]
+ by_id = {node.id: node for node in nodes}
+ community_inputs = [
+ (
+ index,
+ sorted(group),
+ [
+ {
+ "name": by_id[node_id].canonical_name,
+ "type": by_id[node_id].entity_type,
+ "description": by_id[node_id].description,
+ }
+ for node_id in sorted(group)
+ ],
+ )
+ for index, group in enumerate(groups, start=1)
+ ]
+
+ summaries: list[tuple[int, list[int], str, str]] = []
+ for index, node_ids, evidence in community_inputs:
+ summary = context.llm.complete_json(
+ (
+ "Summarize a knowledge-graph community using only the supplied "
+ 'evidence. Return JSON with non-empty "title" and "summary" strings.'
+ ),
+ json.dumps(evidence, ensure_ascii=False),
+ )
+ title = str(summary.get("title", "")).strip()
+ description = str(summary.get("summary", "")).strip()
+ if not title or not description:
+ raise ValueError("community summary requires title and summary")
+ summaries.append((index, node_ids, title, description))
+
+ with context.storage.session() as session:
+ session.execute(
+ delete(Community).where(
+ Community.corpus == context.settings.corpus_name
+ )
+ )
+ for index, node_ids, title, description in summaries:
+ session.add(
+ Community(
+ corpus=context.settings.corpus_name,
+ community_key=f"community-{index}",
+ title=title,
+ summary=description,
+ node_ids_json=node_ids,
+ )
+ )
+ return {"communities": len(groups)}
+
+
+def run_from_args(
+ context: StageContext, args: argparse.Namespace
+) -> dict[str, int]:
+ del args
+ return run(context)
+
+
+if __name__ == "__main__":
+ stage_main(7, run_from_args)
diff --git a/src/08_build_retrieval_index.py b/src/08_build_retrieval_index.py
new file mode 100644
index 0000000..bdf5edb
--- /dev/null
+++ b/src/08_build_retrieval_index.py
@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+import argparse
+from typing import Any
+
+from sqlalchemy import delete, select
+
+from graphrag_platform.cli import stage_main
+from graphrag_platform.models import (
+ Chunk,
+ Community,
+ Document,
+ Edge,
+ Node,
+ SearchItem,
+)
+from graphrag_platform.runtime import StageContext
+
+
+def run(context: StageContext) -> dict[str, int]:
+ context.initialize()
+ with context.storage.session() as session:
+ chunks = session.scalars(
+ select(Chunk).where(Chunk.corpus == context.settings.corpus_name)
+ ).all()
+ nodes = session.scalars(
+ select(Node).where(Node.corpus == context.settings.corpus_name)
+ ).all()
+ edges = session.scalars(
+ select(Edge).where(Edge.corpus == context.settings.corpus_name)
+ ).all()
+ communities = session.scalars(
+ select(Community).where(
+ Community.corpus == context.settings.corpus_name
+ )
+ ).all()
+ if not nodes or not chunks:
+ raise ValueError("stage 8 requires graph nodes and source chunks")
+ documents = {
+ document.id: document
+ for document in session.scalars(
+ select(Document).where(
+ Document.corpus == context.settings.corpus_name
+ )
+ ).all()
+ }
+ node_names = {node.id: node.canonical_name for node in nodes}
+ node_source_chunks = {
+ node.id: set(node.source_chunk_ids_json) for node in nodes
+ }
+ candidates: list[dict[str, Any]] = []
+ for chunk in chunks:
+ document = documents[chunk.document_id]
+ candidates.append(
+ {
+ "item_type": "chunk",
+ "item_key": str(chunk.id),
+ "text": chunk.text,
+ "source_url": document.url,
+ "metadata": {
+ "document_id": document.id,
+ "title": document.title,
+ "chunk_ordinal": chunk.ordinal,
+ },
+ }
+ )
+ for node in nodes:
+ candidates.append(
+ {
+ "item_type": "node",
+ "item_key": str(node.id),
+ "text": (
+ f"{node.canonical_name} ({node.entity_type}): "
+ f"{node.description}"
+ ),
+ "source_url": None,
+ "metadata": {"source_chunk_ids": node.source_chunk_ids_json},
+ }
+ )
+ for edge in edges:
+ candidates.append(
+ {
+ "item_type": "edge",
+ "item_key": str(edge.id),
+ "text": (
+ f"{node_names[edge.source_node_id]} "
+ f"{edge.relationship_type} "
+ f"{node_names[edge.target_node_id]}: {edge.description}"
+ ),
+ "source_url": None,
+ "metadata": {
+ "source_node_id": edge.source_node_id,
+ "target_node_id": edge.target_node_id,
+ "relationship_type": edge.relationship_type,
+ "source_chunk_ids": edge.source_chunk_ids_json,
+ },
+ }
+ )
+ for community in communities:
+ community_chunk_ids: set[int] = set()
+ for node_id in community.node_ids_json:
+ community_chunk_ids.update(node_source_chunks.get(node_id, set()))
+ candidates.append(
+ {
+ "item_type": "community",
+ "item_key": str(community.id),
+ "text": f"{community.title}: {community.summary}",
+ "source_url": None,
+ "metadata": {
+ "node_ids": community.node_ids_json,
+ "source_chunk_ids": sorted(community_chunk_ids),
+ },
+ }
+ )
+ embeddings: list[list[float]] = []
+ for offset in range(0, len(candidates), 64):
+ embeddings.extend(
+ context.llm.embed(
+ [row["text"] for row in candidates[offset : offset + 64]]
+ )
+ )
+ if len(embeddings) != len(candidates):
+ raise ValueError("embedding provider returned an unexpected row count")
+ dimensions = context.settings.llm_embedding_dimensions
+ invalid_dimensions = [
+ len(embedding) for embedding in embeddings if len(embedding) != dimensions
+ ]
+ if invalid_dimensions:
+ raise ValueError(
+ "embedding provider returned vectors with unexpected dimensions: "
+ f"expected {dimensions}, received {invalid_dimensions[0]}"
+ )
+ with context.storage.session() as session:
+ session.execute(
+ delete(SearchItem).where(
+ SearchItem.corpus == context.settings.corpus_name
+ )
+ )
+ for candidate, embedding in zip(candidates, embeddings, strict=True):
+ session.add(
+ SearchItem(
+ corpus=context.settings.corpus_name,
+ item_type=candidate["item_type"],
+ item_key=candidate["item_key"],
+ text=candidate["text"],
+ source_url=candidate["source_url"],
+ metadata_json=candidate["metadata"],
+ embedding=embedding,
+ embedding_model=context.llm.embedding_model,
+ )
+ )
+ context.storage.ensure_vector_index(context.settings.llm_embedding_dimensions)
+ return {"indexed_items": len(candidates)}
+
+
+def run_from_args(
+ context: StageContext, args: argparse.Namespace
+) -> dict[str, int]:
+ del args
+ return run(context)
+
+
+if __name__ == "__main__":
+ stage_main(8, run_from_args)
diff --git a/src/09_query_graphrag.py b/src/09_query_graphrag.py
new file mode 100644
index 0000000..84c8661
--- /dev/null
+++ b/src/09_query_graphrag.py
@@ -0,0 +1,238 @@
+from __future__ import annotations
+
+import argparse
+import math
+from typing import Any
+
+from pgvector.sqlalchemy import Vector
+from sqlalchemy import func, or_, select, text
+from sqlalchemy.orm import Session
+
+from graphrag_platform.cli import stage_main
+from graphrag_platform.models import Chunk, Document, Edge, Node, SearchItem
+from graphrag_platform.runtime import StageContext
+
+
+def cosine_similarity(left: list[float], right: list[float]) -> float:
+ if len(left) != len(right) or not left:
+ return 0.0
+ denominator = math.sqrt(sum(x * x for x in left)) * math.sqrt(
+ sum(x * x for x in right)
+ )
+ if denominator == 0:
+ return 0.0
+ return sum(x * y for x, y in zip(left, right, strict=True)) / denominator
+
+
+def sources_for_chunks(
+ session: Session, chunk_ids: set[int]
+) -> list[dict[str, Any]]:
+ if not chunk_ids:
+ return []
+ rows = session.execute(
+ select(Chunk, Document)
+ .join(Document, Chunk.document_id == Document.id)
+ .where(Chunk.id.in_(chunk_ids))
+ .order_by(Document.url, Chunk.ordinal)
+ ).all()
+ return [
+ {
+ "citation": f"S{index}",
+ "url": document.url,
+ "title": document.title,
+ "chunk_id": chunk.id,
+ "text": chunk.text,
+ }
+ for index, (chunk, document) in enumerate(rows, start=1)
+ ]
+
+
+def rank_vector_context(
+ context: StageContext,
+ session: Session,
+ query_embedding: list[float],
+ limit: int,
+) -> list[tuple[float, SearchItem]]:
+ if context.storage.supports_vector_search:
+ dimensions = context.settings.llm_embedding_dimensions
+ session.execute(
+ text(
+ "SET LOCAL hnsw.ef_search = "
+ f"{context.settings.retrieval_hnsw_ef_search}"
+ )
+ )
+ session.execute(text("SET LOCAL hnsw.iterative_scan = strict_order"))
+ distance = (
+ SearchItem.embedding.cast(Vector(dimensions))
+ .cosine_distance(query_embedding)
+ .label("distance")
+ )
+ rows = session.execute(
+ select(SearchItem, distance)
+ .where(
+ SearchItem.corpus == context.settings.corpus_name,
+ SearchItem.embedding_model == context.llm.embedding_model,
+ func.vector_dims(SearchItem.embedding) == dimensions,
+ )
+ .order_by(distance)
+ .limit(limit)
+ ).all()
+ return [(1.0 - float(distance_value), item) for item, distance_value in rows]
+
+ search_items = session.scalars(
+ select(SearchItem).where(
+ SearchItem.corpus == context.settings.corpus_name,
+ SearchItem.embedding_model == context.llm.embedding_model,
+ )
+ ).all()
+ return sorted(
+ (
+ (cosine_similarity(query_embedding, list(item.embedding)), item)
+ for item in search_items
+ ),
+ key=lambda pair: pair[0],
+ reverse=True,
+ )[:limit]
+
+
+def expand_graph(
+ context: StageContext,
+ session: Session,
+ ranked: list[tuple[float, SearchItem]],
+) -> tuple[list[dict[str, Any]], set[int]]:
+ seed_node_ids: set[int] = set()
+ source_chunk_ids: set[int] = set()
+ for _, item in ranked:
+ source_chunk_ids.update(item.metadata_json.get("source_chunk_ids", []))
+ if item.item_type == "node":
+ seed_node_ids.add(int(item.item_key))
+ elif item.item_type == "edge":
+ seed_node_ids.update(
+ {
+ int(item.metadata_json["source_node_id"]),
+ int(item.metadata_json["target_node_id"]),
+ }
+ )
+ elif item.item_type == "community":
+ seed_node_ids.update(item.metadata_json.get("node_ids", []))
+
+ if not seed_node_ids or context.settings.retrieval_graph_hops == 0:
+ return [], source_chunk_ids
+
+ visited_nodes = set(seed_node_ids)
+ visited_edges: dict[int, Edge] = {}
+ frontier = set(seed_node_ids)
+ for _ in range(context.settings.retrieval_graph_hops):
+ if not frontier or len(visited_edges) >= context.settings.retrieval_graph_limit:
+ break
+ remaining = context.settings.retrieval_graph_limit - len(visited_edges)
+ edges = session.scalars(
+ select(Edge)
+ .where(
+ Edge.corpus == context.settings.corpus_name,
+ or_(
+ Edge.source_node_id.in_(frontier),
+ Edge.target_node_id.in_(frontier),
+ ),
+ )
+ .order_by(Edge.weight.desc(), Edge.id)
+ .limit(remaining)
+ ).all()
+ next_frontier: set[int] = set()
+ for edge in edges:
+ visited_edges[edge.id] = edge
+ source_chunk_ids.update(edge.source_chunk_ids_json)
+ next_frontier.update({edge.source_node_id, edge.target_node_id})
+ next_frontier -= visited_nodes
+ visited_nodes.update(next_frontier)
+ frontier = next_frontier
+
+ nodes = {
+ node.id: node
+ for node in session.scalars(
+ select(Node).where(
+ Node.corpus == context.settings.corpus_name,
+ Node.id.in_(visited_nodes),
+ )
+ ).all()
+ }
+ for node in nodes.values():
+ source_chunk_ids.update(node.source_chunk_ids_json)
+ graph_context = [
+ {
+ "edge_id": edge.id,
+ "source_node_id": edge.source_node_id,
+ "source": nodes[edge.source_node_id].canonical_name,
+ "relationship": edge.relationship_type,
+ "target_node_id": edge.target_node_id,
+ "target": nodes[edge.target_node_id].canonical_name,
+ "description": edge.description,
+ "weight": edge.weight,
+ "source_chunk_ids": edge.source_chunk_ids_json,
+ }
+ for edge in visited_edges.values()
+ if edge.source_node_id in nodes and edge.target_node_id in nodes
+ ]
+ return graph_context, source_chunk_ids
+
+
+def run(
+ context: StageContext,
+ *,
+ question: str,
+ top_k: int | None = None,
+) -> dict[str, Any]:
+ context.initialize()
+ if not question.strip():
+ raise ValueError("question must not be empty")
+ query_embedding = context.llm.embed([question])[0]
+ expected_dimensions = context.settings.llm_embedding_dimensions
+ if len(query_embedding) != expected_dimensions:
+ raise ValueError(
+ "embedding provider returned a query vector with unexpected dimensions: "
+ f"expected {expected_dimensions}, received {len(query_embedding)}"
+ )
+ with context.storage.session() as session:
+ limit = top_k or context.settings.retrieval_top_k
+ ranked = rank_vector_context(
+ context,
+ session,
+ query_embedding,
+ limit,
+ )
+ if not ranked:
+ raise ValueError("stage 9 requires a retrieval index")
+ graph_context, chunk_ids = expand_graph(context, session, ranked)
+ for _, item in ranked:
+ if item.item_type == "chunk":
+ chunk_ids.add(int(item.item_key))
+ chunk_ids.update(item.metadata_json.get("source_chunk_ids", []))
+ sources = sources_for_chunks(session, chunk_ids)
+ return {
+ "question": question,
+ "context": [
+ {
+ "score": round(score, 6),
+ "type": item.item_type,
+ "key": item.item_key,
+ "text": item.text,
+ "source_url": item.source_url,
+ "metadata": item.metadata_json,
+ }
+ for score, item in ranked
+ ],
+ "graph_context": graph_context,
+ "sources": sources,
+ }
+
+
+def run_from_args(
+ context: StageContext, args: argparse.Namespace
+) -> dict[str, Any]:
+ if not args.question:
+ raise ValueError("--question is required for stage 9")
+ return run(context, question=args.question, top_k=args.top_k)
+
+
+if __name__ == "__main__":
+ stage_main(9, run_from_args)
diff --git a/src/10_generate_answer.py b/src/10_generate_answer.py
new file mode 100644
index 0000000..dd6c26c
--- /dev/null
+++ b/src/10_generate_answer.py
@@ -0,0 +1,47 @@
+from __future__ import annotations
+
+import argparse
+import json
+from typing import Any
+
+from graphrag_platform.cli import stage_main
+from graphrag_platform.runtime import StageContext
+from graphrag_platform.stage_loader import load_stage
+
+
+def run(
+ context: StageContext,
+ *,
+ question: str,
+ top_k: int | None = None,
+) -> dict[str, Any]:
+ retrieval = load_stage(9).run(
+ context,
+ question=question,
+ top_k=top_k,
+ )
+ answer = context.llm.complete_text(
+ (
+ "Answer only from the supplied GraphRAG context. Cite supporting sources "
+ "using [S1], [S2], and so on. If the context does not support an answer, "
+ "state that clearly. Do not invent citations."
+ ),
+ json.dumps(retrieval, ensure_ascii=False),
+ )
+ return {
+ "question": question,
+ "answer": answer,
+ "sources": retrieval["sources"],
+ }
+
+
+def run_from_args(
+ context: StageContext, args: argparse.Namespace
+) -> dict[str, Any]:
+ if not args.question:
+ raise ValueError("--question is required for stage 10")
+ return run(context, question=args.question, top_k=args.top_k)
+
+
+if __name__ == "__main__":
+ stage_main(10, run_from_args)
diff --git a/src/graphrag_platform/__init__.py b/src/graphrag_platform/__init__.py
new file mode 100644
index 0000000..b5ea6eb
--- /dev/null
+++ b/src/graphrag_platform/__init__.py
@@ -0,0 +1,6 @@
+"""KnowledgeGraph GraphRAG platform."""
+
+from graphrag_platform.config import Settings
+from graphrag_platform.pipeline import KnowledgeGraphPipeline
+
+__all__ = ["KnowledgeGraphPipeline", "Settings"]
\ No newline at end of file
diff --git a/src/graphrag_platform/api.py b/src/graphrag_platform/api.py
new file mode 100644
index 0000000..e0de338
--- /dev/null
+++ b/src/graphrag_platform/api.py
@@ -0,0 +1,247 @@
+from __future__ import annotations
+
+from typing import Annotated, Any, Callable
+
+import jwt
+from fastapi import Depends, FastAPI, HTTPException, Request, status
+from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
+from pydantic import BaseModel, Field
+from sqlalchemy import select, text
+
+from graphrag_platform.auth import Permission, Principal, decode_internal_token
+from graphrag_platform.config import Settings
+from graphrag_platform.llm import LLMClient
+from graphrag_platform.models import PipelineRun, StageRun
+from graphrag_platform.observability import configure_logging
+from graphrag_platform.orchestrator import requeue_failed_run, submit_run
+from graphrag_platform.runtime import StageContext
+from graphrag_platform.stage_loader import load_stage
+from graphrag_platform.storage import Storage
+
+bearer = HTTPBearer(auto_error=False)
+
+
+class JobRequest(BaseModel):
+ corpus: str = Field(min_length=1, max_length=200)
+ from_stage: int = Field(default=1, ge=1, le=8)
+ through_stage: int = Field(default=8, ge=1, le=8)
+ input_jsonl: str | None = None
+ model_file: str | None = None
+ reset: bool = False
+
+
+class QueryRequest(BaseModel):
+ corpus: str = Field(min_length=1, max_length=200)
+ question: str = Field(min_length=1)
+ top_k: int | None = Field(default=None, ge=1)
+
+
+def _principal(
+ request: Request,
+ credentials: Annotated[
+ HTTPAuthorizationCredentials | None, Depends(bearer)
+ ],
+) -> Principal:
+ if credentials is None:
+ raise HTTPException(
+ status_code=status.HTTP_401_UNAUTHORIZED,
+ detail="Bearer service token is required",
+ )
+ try:
+ return decode_internal_token(credentials.credentials, request.app.state.settings)
+ except jwt.PyJWTError as error:
+ raise HTTPException(
+ status_code=status.HTTP_401_UNAUTHORIZED,
+ detail="Invalid service token",
+ ) from error
+
+
+def require_permissions(
+ required: Permission,
+) -> Callable[[Principal], Principal]:
+ def dependency(principal: Annotated[Principal, Depends(_principal)]) -> Principal:
+ if not principal.has(required):
+ raise HTTPException(
+ status_code=status.HTTP_403_FORBIDDEN,
+ detail=f"KnowledgeGraph permission mask {int(required)} is required",
+ )
+ return principal
+
+ return dependency
+
+
+def create_app(
+ settings: Settings | None = None,
+ *,
+ llm: LLMClient | None = None,
+) -> FastAPI:
+ active_settings = settings or Settings()
+ configure_logging(active_settings.log_level)
+ app = FastAPI(
+ title="KnowledgeGraph Internal API",
+ version="1.0.0",
+ docs_url=None,
+ redoc_url=None,
+ openapi_url="/internal/knowledge-graph/openapi.json",
+ )
+ app.state.settings = active_settings
+ app.state.storage = Storage(active_settings.database_url)
+ app.state.llm = llm
+
+ def context(corpus: str) -> StageContext:
+ corpus_settings = active_settings.model_copy(update={"corpus_name": corpus})
+ return StageContext.create(
+ corpus_settings,
+ storage=app.state.storage,
+ llm=app.state.llm,
+ )
+
+ @app.get("/internal/knowledge-graph/health")
+ def health() -> dict[str, str]:
+ return {"status": "healthy"}
+
+ @app.get("/internal/knowledge-graph/ready")
+ def ready() -> dict[str, str]:
+ try:
+ with app.state.storage.engine.connect() as connection:
+ connection.execute(text("SELECT 1"))
+ if connection.dialect.name == "postgresql":
+ extension = connection.scalar(
+ text(
+ "SELECT extname FROM pg_extension "
+ "WHERE extname = 'vector'"
+ )
+ )
+ if extension != "vector":
+ raise RuntimeError("PgVector extension is not installed")
+ except Exception as error:
+ raise HTTPException(
+ status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
+ detail="Service dependencies are not ready",
+ ) from error
+ return {"status": "ready"}
+
+ @app.post(
+ "/internal/knowledge-graph/jobs",
+ status_code=status.HTTP_202_ACCEPTED,
+ )
+ def create_job(
+ body: JobRequest,
+ principal: Annotated[
+ Principal, Depends(require_permissions(Permission.EXECUTE))
+ ],
+ ) -> dict[str, str]:
+ if body.from_stage == 1 and not principal.has(Permission.ADD):
+ raise HTTPException(
+ status_code=status.HTTP_403_FORBIDDEN,
+ detail="KnowledgeGraph Add permission is required for stage 1",
+ )
+ if body.from_stage > body.through_stage:
+ raise HTTPException(
+ status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
+ detail="from_stage must not exceed through_stage",
+ )
+ run_context = context(body.corpus)
+ run_context.initialize()
+ run_id = submit_run(
+ run_context,
+ from_stage=body.from_stage,
+ through_stage=body.through_stage,
+ payload={
+ "input_jsonl": body.input_jsonl,
+ "model_file": body.model_file,
+ "reset": body.reset,
+ },
+ requested_by=principal.subject,
+ correlation_id=principal.correlation_id,
+ permission_mask=principal.permission_mask,
+ )
+ return {"job_id": run_id, "status": "queued"}
+
+ @app.get("/internal/knowledge-graph/jobs/{job_id}")
+ def get_job(
+ job_id: str,
+ principal: Annotated[
+ Principal, Depends(require_permissions(Permission.READ))
+ ],
+ ) -> dict[str, Any]:
+ del principal
+ with app.state.storage.session() as session:
+ run = session.get(PipelineRun, job_id)
+ if run is None:
+ raise HTTPException(status_code=404, detail="Job not found")
+ stages = session.scalars(
+ select(StageRun)
+ .where(StageRun.pipeline_run_id == job_id)
+ .order_by(StageRun.stage_number, StageRun.attempt)
+ ).all()
+ return {
+ "job_id": run.id,
+ "corpus": run.corpus,
+ "status": run.status,
+ "attempt": run.attempt,
+ "from_stage": run.from_stage,
+ "through_stage": run.through_stage,
+ "error": run.error_message,
+ "output": run.output_json,
+ "stages": [
+ {
+ "stage": stage_run.stage_number,
+ "name": stage_run.stage_name,
+ "status": stage_run.status,
+ "attempt": stage_run.attempt,
+ "output": stage_run.output_json,
+ "error": stage_run.error_message,
+ }
+ for stage_run in stages
+ ],
+ }
+
+ @app.post(
+ "/internal/knowledge-graph/jobs/{job_id}/retry",
+ status_code=status.HTTP_202_ACCEPTED,
+ )
+ def retry_job(
+ job_id: str,
+ principal: Annotated[
+ Principal, Depends(require_permissions(Permission.EXECUTE))
+ ],
+ ) -> dict[str, str]:
+ del principal
+ requeue_failed_run(context(active_settings.corpus_name), job_id)
+ return {"job_id": job_id, "status": "queued"}
+
+ @app.post("/internal/knowledge-graph/query")
+ def query(
+ body: QueryRequest,
+ principal: Annotated[
+ Principal,
+ Depends(require_permissions(Permission.READ | Permission.EXECUTE)),
+ ],
+ ) -> dict[str, Any]:
+ del principal
+ return load_stage(9).run(
+ context(body.corpus),
+ question=body.question,
+ top_k=body.top_k,
+ )
+
+ @app.post("/internal/knowledge-graph/answer")
+ def answer(
+ body: QueryRequest,
+ principal: Annotated[
+ Principal,
+ Depends(require_permissions(Permission.READ | Permission.EXECUTE)),
+ ],
+ ) -> dict[str, Any]:
+ del principal
+ return load_stage(10).run(
+ context(body.corpus),
+ question=body.question,
+ top_k=body.top_k,
+ )
+
+ return app
+
+
+app = create_app()
diff --git a/src/graphrag_platform/auth.py b/src/graphrag_platform/auth.py
new file mode 100644
index 0000000..db58a83
--- /dev/null
+++ b/src/graphrag_platform/auth.py
@@ -0,0 +1,56 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import IntFlag
+from typing import Any
+
+import jwt
+
+from graphrag_platform.config import Settings
+
+
+class Permission(IntFlag):
+ READ = 1
+ ADD = 2
+ EDIT = 4
+ DELETE = 8
+ EXECUTE = 16
+
+
+@dataclass(frozen=True)
+class Principal:
+ subject: str
+ permission_mask: int
+ correlation_id: str
+ claims: dict[str, Any]
+
+ def has(self, required: Permission) -> bool:
+ return (self.permission_mask & int(required)) == int(required)
+
+
+def decode_internal_token(token: str, settings: Settings) -> Principal:
+ claims = jwt.decode(
+ token,
+ settings.internal_jwt_secret,
+ algorithms=["HS256"],
+ issuer=settings.internal_jwt_issuer,
+ audience=settings.internal_jwt_audience,
+ options={"require": ["exp", "iat", "iss", "aud", "sub", "features"]},
+ )
+ features = claims.get("features")
+ if not isinstance(features, dict):
+ raise jwt.InvalidTokenError("features claim must be an object")
+ raw_mask = features.get(settings.internal_feature_name)
+ if not isinstance(raw_mask, int):
+ raise jwt.InvalidTokenError(
+ f"missing integer feature claim: {settings.internal_feature_name}"
+ )
+ correlation_id = str(claims.get("correlation_id") or claims.get("jti") or "")
+ if not correlation_id:
+ raise jwt.InvalidTokenError("correlation_id or jti is required")
+ return Principal(
+ subject=str(claims["sub"]),
+ permission_mask=raw_mask,
+ correlation_id=correlation_id,
+ claims=claims,
+ )
diff --git a/src/graphrag_platform/cli.py b/src/graphrag_platform/cli.py
new file mode 100644
index 0000000..331e9ae
--- /dev/null
+++ b/src/graphrag_platform/cli.py
@@ -0,0 +1,111 @@
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from uuid import uuid4
+from collections.abc import Callable
+from pathlib import Path
+from typing import Any
+
+from graphrag_platform.config import Settings
+from graphrag_platform.observability import configure_logging
+from graphrag_platform.orchestrator import execute_run, submit_run
+from graphrag_platform.runtime import StageContext
+from graphrag_platform.stage_loader import STAGE_MODULES, load_stage
+
+StageRunner = Callable[[StageContext, argparse.Namespace], dict[str, Any]]
+
+
+def create_parser(fixed_stage: int | None) -> argparse.ArgumentParser:
+ parser = argparse.ArgumentParser(
+ description="Run a KnowledgeGraph GraphRAG pipeline stage."
+ )
+ if fixed_stage is None:
+ parser.add_argument("stage", type=int, choices=range(1, 11))
+ parser.add_argument("--env-file", type=Path, default=Path(".env"))
+ parser.add_argument("--database-url", help="Override DATABASE_URL.")
+ parser.add_argument("--corpus", help="Override CORPUS_NAME.")
+ parser.add_argument("--input-jsonl", type=Path)
+ parser.add_argument("--model-file", type=Path)
+ parser.add_argument("--question")
+ parser.add_argument("--top-k", type=int)
+ parser.add_argument("--reset", action="store_true")
+ return parser
+
+
+def create_run_parser() -> argparse.ArgumentParser:
+ parser = create_parser(1)
+ parser.description = "Run a durable KnowledgeGraph build pipeline."
+ parser.add_argument("--from-stage", type=int, choices=range(1, 9), default=1)
+ parser.add_argument("--through-stage", type=int, choices=range(1, 9), default=8)
+ parser.add_argument("--requested-by", default="local-cli")
+ parser.add_argument("--correlation-id", default=None)
+ return parser
+
+
+def create_context(
+ args: argparse.Namespace, stage: int, *, apply_reset: bool = True
+) -> StageContext:
+ overrides: dict[str, Any] = {}
+ if args.database_url:
+ overrides["database_url"] = args.database_url
+ if args.corpus:
+ overrides["corpus_name"] = args.corpus
+ settings = Settings(_env_file=args.env_file, **overrides)
+ context = StageContext.create(settings)
+ context.initialize()
+ if args.reset and apply_reset:
+ context.storage.clear_from_stage(settings.corpus_name, stage)
+ return context
+
+
+def run(stage: int, args: argparse.Namespace) -> dict[str, Any]:
+ context = create_context(args, stage)
+ module = load_stage(stage)
+ return module.run_from_args(context, args)
+
+
+def stage_main(stage: int, runner: StageRunner) -> None:
+ args = create_parser(stage).parse_args()
+ context = create_context(args, stage)
+ configure_logging(context.settings.log_level)
+ result = runner(context, args)
+ print(json.dumps(result, indent=2, ensure_ascii=False))
+
+
+def main_all() -> None:
+ if len(sys.argv) > 1 and sys.argv[1] == "run":
+ args = create_run_parser().parse_args(sys.argv[2:])
+ context = create_context(args, args.from_stage, apply_reset=False)
+ configure_logging(context.settings.log_level)
+ run_id = submit_run(
+ context,
+ from_stage=args.from_stage,
+ through_stage=args.through_stage,
+ payload={
+ "input_jsonl": str(args.input_jsonl) if args.input_jsonl else None,
+ "model_file": str(args.model_file) if args.model_file else None,
+ "reset": args.reset,
+ },
+ requested_by=args.requested_by,
+ correlation_id=args.correlation_id or str(uuid4()),
+ permission_mask=31,
+ )
+ outputs = execute_run(context, run_id)
+ print(json.dumps({"run_id": run_id, "outputs": outputs}, indent=2))
+ return
+ args = create_parser(None).parse_args()
+ configure_logging(Settings(_env_file=args.env_file).log_level)
+ result = run(args.stage, args)
+ print(
+ json.dumps(
+ {
+ "stage": args.stage,
+ "name": STAGE_MODULES[args.stage].removeprefix(f"{args.stage:02d}_"),
+ **result,
+ },
+ indent=2,
+ ensure_ascii=False,
+ )
+ )
diff --git a/src/graphrag_platform/config.py b/src/graphrag_platform/config.py
new file mode 100644
index 0000000..2930fe4
--- /dev/null
+++ b/src/graphrag_platform/config.py
@@ -0,0 +1,126 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+from typing import Annotated, Any
+
+from pydantic import Field, field_validator
+from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
+
+
+class Settings(BaseSettings):
+ """Runtime configuration loaded from environment variables or a .env file."""
+
+ model_config = SettingsConfigDict(
+ env_file=".env",
+ env_file_encoding="utf-8",
+ extra="ignore",
+ case_sensitive=False,
+ )
+
+ database_url: str = (
+ "postgresql+psycopg://knowledgegraph_app:change-me"
+ "@localhost:5432/knowledgegraph"
+ )
+ source_database_url: str | None = None
+ source_table_name: str = "external_documents"
+ analytics_database_url: str | None = None
+ auto_create_schema: bool = False
+
+ corpus_name: str = "default-corpus"
+
+ entity_types: Annotated[list[str], NoDecode] = Field(
+ default_factory=lambda: [
+ "Person",
+ "Organization",
+ "Product",
+ "Concept",
+ "Event",
+ "Location",
+ ]
+ )
+ relationship_types: Annotated[list[str], NoDecode] = Field(
+ default_factory=lambda: [
+ "AFFILIATED_WITH",
+ "CREATED",
+ "DEPENDS_ON",
+ "MENTIONS",
+ "PART_OF",
+ "RELATED_TO",
+ ]
+ )
+ node_template: dict[str, Any] = Field(
+ default_factory=lambda: {
+ "canonical_name": "string",
+ "entity_type": "one configured entity type",
+ "description": "source-grounded string",
+ "aliases": ["string"],
+ }
+ )
+ edge_template: dict[str, Any] = Field(
+ default_factory=lambda: {
+ "source": "entity name",
+ "target": "entity name",
+ "relationship_type": "one configured relationship type",
+ "description": "source-grounded string",
+ }
+ )
+ extraction_instructions: str = (
+ "Extract only entities and relationships explicitly supported by the supplied text."
+ )
+
+ llm_base_url: str = "https://api.openai.com/v1"
+ llm_api_key: str | None = None
+ llm_chat_model: str = "gpt-4.1-mini"
+ llm_embedding_model: str = "text-embedding-3-small"
+ llm_embedding_dimensions: int = Field(default=1536, ge=1)
+ llm_timeout_seconds: float = Field(default=60, gt=0)
+ llm_max_retries: int = Field(default=4, ge=0, le=10)
+ llm_retry_initial_seconds: float = Field(default=1, gt=0)
+ llm_retry_max_seconds: float = Field(default=30, gt=0)
+
+ chunk_size: int = Field(default=1200, ge=200)
+ chunk_overlap: int = Field(default=150, ge=0)
+ community_resolution: float = Field(default=1.0, gt=0)
+ retrieval_top_k: int = Field(default=8, ge=1)
+ retrieval_graph_hops: int = Field(default=2, ge=0, le=5)
+ retrieval_graph_limit: int = Field(default=50, ge=1)
+ retrieval_hnsw_ef_search: int = Field(default=100, ge=1)
+ log_level: str = "INFO"
+ internal_jwt_secret: str = "replace-in-dev-test"
+ internal_jwt_issuer: str = "knowledgegraph-bff"
+ internal_jwt_audience: str = "knowledgegraph-service"
+ internal_feature_name: str = "KnowledgeGraph"
+ worker_poll_seconds: float = Field(default=2, gt=0)
+ worker_stale_run_seconds: int = Field(default=3600, ge=60)
+ artifact_dir: Path = Path("artifacts")
+
+ @field_validator(
+ "entity_types",
+ "relationship_types",
+ mode="before",
+ )
+ @classmethod
+ def parse_comma_separated_list(cls, value: object) -> object:
+ if isinstance(value, str):
+ return [item.strip() for item in value.split(",") if item.strip()]
+ return value
+
+ @field_validator("node_template", "edge_template", mode="before")
+ @classmethod
+ def parse_json_object(cls, value: object) -> object:
+ if isinstance(value, str):
+ parsed = json.loads(value)
+ if not isinstance(parsed, dict):
+ raise ValueError("template must be a JSON object")
+ return parsed
+ return value
+
+ def model_definition(self) -> dict[str, Any]:
+ return {
+ "entity_types": self.entity_types,
+ "relationship_types": self.relationship_types,
+ "node_template": self.node_template,
+ "edge_template": self.edge_template,
+ "extraction_instructions": self.extraction_instructions,
+ }
\ No newline at end of file
diff --git a/src/graphrag_platform/llm.py b/src/graphrag_platform/llm.py
new file mode 100644
index 0000000..8fe00ce
--- /dev/null
+++ b/src/graphrag_platform/llm.py
@@ -0,0 +1,153 @@
+from __future__ import annotations
+
+import json
+import logging
+import random
+import time
+from collections.abc import Sequence
+from typing import Any, Callable, Protocol
+
+import httpx
+
+from graphrag_platform.config import Settings
+
+LOGGER = logging.getLogger(__name__)
+_RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
+
+
+class LLMClient(Protocol):
+ chat_model: str
+ embedding_model: str
+
+ def complete_json(self, system: str, user: str) -> dict[str, Any]: ...
+
+ def complete_text(self, system: str, user: str) -> str: ...
+
+ def embed(self, texts: Sequence[str]) -> list[list[float]]: ...
+
+
+class OpenAICompatibleClient:
+ def __init__(
+ self,
+ settings: Settings,
+ transport: httpx.BaseTransport | None = None,
+ sleeper: Callable[[float], None] = time.sleep,
+ jitter: Callable[[], float] = random.random,
+ ) -> None:
+ if not settings.llm_api_key:
+ raise ValueError("LLM_API_KEY is required for LLM-backed stages")
+ self.chat_model = settings.llm_chat_model
+ self.embedding_model = settings.llm_embedding_model
+ self._settings = settings
+ self._sleeper = sleeper
+ self._jitter = jitter
+ self._client = httpx.Client(
+ base_url=settings.llm_base_url.rstrip("/") + "/",
+ timeout=settings.llm_timeout_seconds,
+ headers={
+ "Authorization": f"Bearer {settings.llm_api_key}",
+ "Content-Type": "application/json",
+ },
+ transport=transport,
+ )
+
+ def complete_json(self, system: str, user: str) -> dict[str, Any]:
+ response = self._post(
+ "chat/completions",
+ json={
+ "model": self.chat_model,
+ "response_format": {"type": "json_object"},
+ "temperature": 0,
+ "messages": [
+ {"role": "system", "content": system},
+ {"role": "user", "content": user},
+ ],
+ },
+ )
+ content = response.json()["choices"][0]["message"]["content"]
+ parsed = json.loads(content)
+ if not isinstance(parsed, dict):
+ raise ValueError("LLM response must be a JSON object")
+ return parsed
+
+ def complete_text(self, system: str, user: str) -> str:
+ response = self._post(
+ "chat/completions",
+ json={
+ "model": self.chat_model,
+ "temperature": 0,
+ "messages": [
+ {"role": "system", "content": system},
+ {"role": "user", "content": user},
+ ],
+ },
+ )
+ return str(response.json()["choices"][0]["message"]["content"]).strip()
+
+ def embed(self, texts: Sequence[str]) -> list[list[float]]:
+ if not texts:
+ return []
+ response = self._post(
+ "embeddings",
+ json={"model": self.embedding_model, "input": list(texts)},
+ )
+ rows = sorted(response.json()["data"], key=lambda row: row["index"])
+ return [[float(value) for value in row["embedding"]] for row in rows]
+
+ def _post(self, path: str, *, json: dict[str, Any]) -> httpx.Response:
+ attempts = self._settings.llm_max_retries + 1
+ for attempt in range(1, attempts + 1):
+ try:
+ response = self._client.post(path, json=json)
+ if response.status_code not in _RETRYABLE_STATUS_CODES:
+ response.raise_for_status()
+ return response
+ if attempt == attempts:
+ response.raise_for_status()
+ delay = self._retry_delay(response, attempt)
+ LOGGER.warning(
+ "Retrying model provider request",
+ extra={
+ "event": "llm_retry",
+ "attempt": attempt,
+ "status_code": response.status_code,
+ "delay_seconds": delay,
+ "path": path,
+ },
+ )
+ self._sleeper(delay)
+ except (httpx.ConnectError, httpx.TimeoutException) as error:
+ if attempt == attempts:
+ raise
+ delay = self._retry_delay(None, attempt)
+ LOGGER.warning(
+ "Retrying model provider request after transport error",
+ extra={
+ "event": "llm_retry",
+ "attempt": attempt,
+ "error_type": type(error).__name__,
+ "delay_seconds": delay,
+ "path": path,
+ },
+ )
+ self._sleeper(delay)
+ raise RuntimeError("model provider retry loop ended unexpectedly")
+
+ def _retry_delay(
+ self, response: httpx.Response | None, attempt: int
+ ) -> float:
+ if response is not None:
+ retry_after = response.headers.get("Retry-After")
+ if retry_after:
+ try:
+ return min(
+ float(retry_after),
+ self._settings.llm_retry_max_seconds,
+ )
+ except ValueError:
+ pass
+ exponential = self._settings.llm_retry_initial_seconds * (2 ** (attempt - 1))
+ return min(
+ exponential + self._jitter(),
+ self._settings.llm_retry_max_seconds,
+ )
\ No newline at end of file
diff --git a/src/graphrag_platform/models.py b/src/graphrag_platform/models.py
new file mode 100644
index 0000000..ad219ad
--- /dev/null
+++ b/src/graphrag_platform/models.py
@@ -0,0 +1,281 @@
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+from uuid import uuid4
+
+from sqlalchemy import (
+ DateTime,
+ Float,
+ ForeignKey,
+ Index,
+ Integer,
+ JSON,
+ String,
+ Text,
+ UniqueConstraint,
+)
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+from pgvector.sqlalchemy import Vector
+
+
+def utc_now() -> datetime:
+ return datetime.now(timezone.utc)
+
+
+class Base(DeclarativeBase):
+ pass
+
+
+class RawDocument(Base):
+ __tablename__ = "raw_documents"
+ __table_args__ = (
+ UniqueConstraint("corpus", "external_id", name="uq_raw_corpus_external_id"),
+ )
+
+ id: Mapped[int] = mapped_column(primary_key=True)
+ corpus: Mapped[str] = mapped_column(String(200), index=True)
+ external_id: Mapped[str] = mapped_column(String(500))
+ source_schema_version: Mapped[str] = mapped_column(String(20))
+ url: Mapped[str] = mapped_column(Text)
+ content_type: Mapped[str | None] = mapped_column(String(200))
+ status_code: Mapped[int | None] = mapped_column(Integer)
+ body: Mapped[str] = mapped_column(Text)
+ metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
+ retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
+ collected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
+
+
+class Document(Base):
+ __tablename__ = "documents"
+ __table_args__ = (
+ UniqueConstraint("corpus", "url", name="uq_document_corpus_url"),
+ UniqueConstraint("raw_document_id", name="uq_document_raw_document"),
+ )
+
+ id: Mapped[int] = mapped_column(primary_key=True)
+ raw_document_id: Mapped[int | None] = mapped_column(
+ ForeignKey("raw_documents.id", ondelete="SET NULL")
+ )
+ corpus: Mapped[str] = mapped_column(String(200), index=True)
+ url: Mapped[str] = mapped_column(Text)
+ title: Mapped[str | None] = mapped_column(Text)
+ author: Mapped[str | None] = mapped_column(Text)
+ published_at: Mapped[str | None] = mapped_column(String(100))
+ text: Mapped[str] = mapped_column(Text)
+ content_hash: Mapped[str] = mapped_column(String(64), index=True)
+ metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
+ is_duplicate: Mapped[bool] = mapped_column(default=False)
+ canonical_document_id: Mapped[int | None] = mapped_column(
+ ForeignKey("documents.id", ondelete="SET NULL")
+ )
+
+
+class Chunk(Base):
+ __tablename__ = "chunks"
+ __table_args__ = (UniqueConstraint("document_id", "ordinal", name="uq_chunk_ordinal"),)
+
+ id: Mapped[int] = mapped_column(primary_key=True)
+ document_id: Mapped[int] = mapped_column(
+ ForeignKey("documents.id", ondelete="CASCADE"), index=True
+ )
+ corpus: Mapped[str] = mapped_column(String(200), index=True)
+ ordinal: Mapped[int] = mapped_column(Integer)
+ text: Mapped[str] = mapped_column(Text)
+ content_hash: Mapped[str] = mapped_column(String(64), index=True)
+
+
+class KnowledgeModel(Base):
+ __tablename__ = "knowledge_models"
+
+ corpus: Mapped[str] = mapped_column(String(200), primary_key=True)
+ definition_json: Mapped[dict[str, Any]] = mapped_column(JSON)
+ updated_at: Mapped[datetime] = mapped_column(
+ DateTime(timezone=True), default=utc_now, onupdate=utc_now
+ )
+
+
+class Extraction(Base):
+ __tablename__ = "extractions"
+ __table_args__ = (UniqueConstraint("chunk_id", name="uq_extraction_chunk"),)
+
+ id: Mapped[int] = mapped_column(primary_key=True)
+ chunk_id: Mapped[int] = mapped_column(
+ ForeignKey("chunks.id", ondelete="CASCADE"), index=True
+ )
+ corpus: Mapped[str] = mapped_column(String(200), index=True)
+ payload_json: Mapped[dict[str, Any]] = mapped_column(JSON)
+ model_name: Mapped[str] = mapped_column(String(200))
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
+
+
+class ResolvedEntity(Base):
+ __tablename__ = "resolved_entities"
+ __table_args__ = (
+ UniqueConstraint("corpus", "canonical_key", name="uq_resolved_entity_key"),
+ )
+
+ id: Mapped[int] = mapped_column(primary_key=True)
+ corpus: Mapped[str] = mapped_column(String(200), index=True)
+ canonical_key: Mapped[str] = mapped_column(String(500))
+ canonical_name: Mapped[str] = mapped_column(Text)
+ entity_type: Mapped[str] = mapped_column(String(100))
+ description: Mapped[str] = mapped_column(Text, default="")
+ aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list)
+ source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
+
+
+class ResolvedRelationship(Base):
+ __tablename__ = "resolved_relationships"
+ __table_args__ = (
+ UniqueConstraint(
+ "corpus",
+ "source_key",
+ "target_key",
+ "relationship_type",
+ name="uq_resolved_relationship",
+ ),
+ )
+
+ id: Mapped[int] = mapped_column(primary_key=True)
+ corpus: Mapped[str] = mapped_column(String(200), index=True)
+ source_key: Mapped[str] = mapped_column(String(500))
+ target_key: Mapped[str] = mapped_column(String(500))
+ relationship_type: Mapped[str] = mapped_column(String(100))
+ description: Mapped[str] = mapped_column(Text, default="")
+ source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
+
+
+class Node(Base):
+ __tablename__ = "nodes"
+ __table_args__ = (
+ UniqueConstraint("corpus", "canonical_key", name="uq_node_canonical_key"),
+ )
+
+ id: Mapped[int] = mapped_column(primary_key=True)
+ corpus: Mapped[str] = mapped_column(String(200), index=True)
+ canonical_key: Mapped[str] = mapped_column(String(500))
+ canonical_name: Mapped[str] = mapped_column(Text)
+ entity_type: Mapped[str] = mapped_column(String(100), index=True)
+ description: Mapped[str] = mapped_column(Text, default="")
+ aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list)
+ source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
+
+
+class Edge(Base):
+ __tablename__ = "edges"
+ __table_args__ = (
+ UniqueConstraint(
+ "corpus",
+ "source_node_id",
+ "target_node_id",
+ "relationship_type",
+ name="uq_edge_identity",
+ ),
+ )
+
+ id: Mapped[int] = mapped_column(primary_key=True)
+ corpus: Mapped[str] = mapped_column(String(200), index=True)
+ source_node_id: Mapped[int] = mapped_column(
+ ForeignKey("nodes.id", ondelete="CASCADE"), index=True
+ )
+ target_node_id: Mapped[int] = mapped_column(
+ ForeignKey("nodes.id", ondelete="CASCADE"), index=True
+ )
+ relationship_type: Mapped[str] = mapped_column(String(100), index=True)
+ description: Mapped[str] = mapped_column(Text, default="")
+ weight: Mapped[float] = mapped_column(Float, default=1.0)
+ source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
+
+
+class Community(Base):
+ __tablename__ = "communities"
+ __table_args__ = (UniqueConstraint("corpus", "community_key", name="uq_community_key"),)
+
+ id: Mapped[int] = mapped_column(primary_key=True)
+ corpus: Mapped[str] = mapped_column(String(200), index=True)
+ community_key: Mapped[str] = mapped_column(String(100))
+ title: Mapped[str] = mapped_column(Text)
+ summary: Mapped[str] = mapped_column(Text)
+ node_ids_json: Mapped[list[int]] = mapped_column(JSON)
+
+
+class SearchItem(Base):
+ __tablename__ = "retrieval_items"
+ __table_args__ = (
+ UniqueConstraint("corpus", "item_type", "item_key", name="uq_search_item"),
+ Index("ix_retrieval_items_corpus_type", "corpus", "item_type"),
+ Index(
+ "ix_retrieval_items_corpus_model",
+ "corpus",
+ "embedding_model",
+ ),
+ )
+
+ id: Mapped[int] = mapped_column(primary_key=True)
+ corpus: Mapped[str] = mapped_column(String(200), index=True)
+ item_type: Mapped[str] = mapped_column(String(50))
+ item_key: Mapped[str] = mapped_column(String(100))
+ text: Mapped[str] = mapped_column(Text)
+ source_url: Mapped[str | None] = mapped_column(Text)
+ metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
+ embedding: Mapped[list[float]] = mapped_column(
+ Vector().with_variant(JSON(), "sqlite")
+ )
+ embedding_model: Mapped[str] = mapped_column(String(200))
+
+
+class PipelineRun(Base):
+ __tablename__ = "pipeline_runs"
+ __table_args__ = (
+ Index("ix_pipeline_runs_status_created", "status", "created_at"),
+ Index("ix_pipeline_runs_corpus_created", "corpus", "created_at"),
+ )
+
+ id: Mapped[str] = mapped_column(
+ String(36), primary_key=True, default=lambda: str(uuid4())
+ )
+ corpus: Mapped[str] = mapped_column(String(200), index=True)
+ status: Mapped[str] = mapped_column(String(30), index=True, default="queued")
+ from_stage: Mapped[int] = mapped_column(Integer)
+ through_stage: Mapped[int] = mapped_column(Integer)
+ payload_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
+ configuration_hash: Mapped[str] = mapped_column(String(64))
+ requested_by: Mapped[str] = mapped_column(String(500))
+ correlation_id: Mapped[str] = mapped_column(String(200), index=True)
+ feature_name: Mapped[str] = mapped_column(String(200), default="KnowledgeGraph")
+ permission_mask: Mapped[int] = mapped_column(Integer)
+ attempt: Mapped[int] = mapped_column(Integer, default=0)
+ output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
+ error_message: Mapped[str | None] = mapped_column(Text)
+ created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
+ started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+ heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
+
+
+class StageRun(Base):
+ __tablename__ = "stage_runs"
+ __table_args__ = (
+ UniqueConstraint(
+ "pipeline_run_id",
+ "stage_number",
+ "attempt",
+ name="uq_stage_run_attempt",
+ ),
+ Index("ix_stage_runs_pipeline_stage", "pipeline_run_id", "stage_number"),
+ )
+
+ id: Mapped[int] = mapped_column(primary_key=True)
+ pipeline_run_id: Mapped[str] = mapped_column(
+ ForeignKey("pipeline_runs.id", ondelete="CASCADE"), index=True
+ )
+ stage_number: Mapped[int] = mapped_column(Integer)
+ stage_name: Mapped[str] = mapped_column(String(200))
+ status: Mapped[str] = mapped_column(String(30), default="running")
+ attempt: Mapped[int] = mapped_column(Integer)
+ input_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
+ output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
+ error_message: Mapped[str | None] = mapped_column(Text)
+ started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
+ finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
\ No newline at end of file
diff --git a/src/graphrag_platform/observability.py b/src/graphrag_platform/observability.py
new file mode 100644
index 0000000..785169e
--- /dev/null
+++ b/src/graphrag_platform/observability.py
@@ -0,0 +1,33 @@
+from __future__ import annotations
+
+import json
+import logging
+from datetime import datetime, timezone
+from typing import Any
+
+
+class JsonFormatter(logging.Formatter):
+ _reserved = set(logging.makeLogRecord({}).__dict__)
+
+ def format(self, record: logging.LogRecord) -> str:
+ payload: dict[str, Any] = {
+ "timestamp": datetime.now(timezone.utc).isoformat(),
+ "level": record.levelname,
+ "logger": record.name,
+ "message": record.getMessage(),
+ }
+ for key, value in record.__dict__.items():
+ if key not in self._reserved and key not in {"message", "asctime"}:
+ payload[key] = value
+ if record.exc_info:
+ payload["exception"] = self.formatException(record.exc_info)
+ return json.dumps(payload, ensure_ascii=False, default=str)
+
+
+def configure_logging(level: str = "INFO") -> None:
+ handler = logging.StreamHandler()
+ handler.setFormatter(JsonFormatter())
+ root = logging.getLogger()
+ root.handlers.clear()
+ root.addHandler(handler)
+ root.setLevel(level.upper())
diff --git a/src/graphrag_platform/orchestrator.py b/src/graphrag_platform/orchestrator.py
new file mode 100644
index 0000000..a3da801
--- /dev/null
+++ b/src/graphrag_platform/orchestrator.py
@@ -0,0 +1,330 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import logging
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+from sqlalchemy import select, text, update
+
+from graphrag_platform.models import PipelineRun, StageRun
+from graphrag_platform.runtime import StageContext
+from graphrag_platform.stage_loader import STAGE_MODULES, load_stage
+
+LOGGER = logging.getLogger(__name__)
+BUILD_STAGES = range(1, 9)
+
+
+class RunDeferredError(RuntimeError):
+ pass
+
+
+def utc_now() -> datetime:
+ return datetime.now(timezone.utc)
+
+
+def configuration_hash(context: StageContext) -> str:
+ payload = context.settings.model_dump(
+ mode="json",
+ exclude={"llm_api_key", "internal_jwt_secret"},
+ )
+ encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"))
+ return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
+
+
+def submit_run(
+ context: StageContext,
+ *,
+ from_stage: int,
+ through_stage: int,
+ payload: dict[str, Any],
+ requested_by: str,
+ correlation_id: str,
+ permission_mask: int,
+) -> str:
+ if from_stage not in BUILD_STAGES or through_stage not in BUILD_STAGES:
+ raise ValueError("pipeline jobs support build stages 1 through 8")
+ if from_stage > through_stage:
+ raise ValueError("from_stage must not exceed through_stage")
+ run = PipelineRun(
+ corpus=context.settings.corpus_name,
+ status="queued",
+ from_stage=from_stage,
+ through_stage=through_stage,
+ payload_json=payload,
+ configuration_hash=configuration_hash(context),
+ requested_by=requested_by,
+ correlation_id=correlation_id,
+ feature_name=context.settings.internal_feature_name,
+ permission_mask=permission_mask,
+ )
+ with context.storage.session() as session:
+ session.add(run)
+ session.flush()
+ run_id = run.id
+ LOGGER.info(
+ "Pipeline run queued",
+ extra={
+ "event": "pipeline_queued",
+ "run_id": run_id,
+ "corpus": context.settings.corpus_name,
+ "from_stage": from_stage,
+ "through_stage": through_stage,
+ "correlation_id": correlation_id,
+ },
+ )
+ return run_id
+
+
+def claim_next_run(context: StageContext) -> str | None:
+ with context.storage.session() as session:
+ statement = (
+ select(PipelineRun)
+ .where(PipelineRun.status == "queued")
+ .order_by(PipelineRun.created_at)
+ .limit(1)
+ )
+ if context.storage.engine.dialect.name == "postgresql":
+ statement = statement.with_for_update(skip_locked=True)
+ run = session.scalar(statement)
+ if run is None:
+ return None
+ run.status = "running"
+ run.attempt += 1
+ run.started_at = run.started_at or utc_now()
+ run.heartbeat_at = utc_now()
+ session.flush()
+ return run.id
+
+
+def execute_run(context: StageContext, run_id: str) -> dict[str, Any]:
+ with context.storage.session() as session:
+ run = session.get(PipelineRun, run_id)
+ if run is None:
+ raise ValueError(f"pipeline run does not exist: {run_id}")
+ if run.status == "queued":
+ run.status = "running"
+ run.attempt += 1
+ run.started_at = run.started_at or utc_now()
+ if run.status != "running":
+ raise ValueError(f"pipeline run {run_id} is {run.status}, not running")
+ from_stage = run.from_stage
+ through_stage = run.through_stage
+ payload = dict(run.payload_json)
+ attempt = run.attempt
+ execution_context = context.for_corpus(run.corpus)
+ corpus = run.corpus
+
+ lock_connection = None
+ if context.storage.engine.dialect.name == "postgresql":
+ lock_connection = context.storage.engine.connect()
+ acquired = lock_connection.scalar(
+ text("SELECT pg_try_advisory_lock(hashtext(:key))"),
+ {"key": f"knowledgegraph:{corpus}"},
+ )
+ if not acquired:
+ lock_connection.close()
+ with context.storage.session() as session:
+ run = session.get(PipelineRun, run_id)
+ if run is not None:
+ run.status = "queued"
+ run.error_message = "Deferred because the corpus is already running"
+ raise RunDeferredError(f"corpus already has an active run: {corpus}")
+
+ outputs: dict[str, Any] = {}
+ try:
+ if payload.get("reset") and attempt == 1:
+ execution_context.storage.clear_from_stage(corpus, from_stage)
+ for stage_number in range(from_stage, through_stage + 1):
+ previous_output = _successful_stage_output(
+ context, run_id, stage_number
+ )
+ if previous_output is not None:
+ outputs[str(stage_number)] = previous_output
+ continue
+ stage_name = STAGE_MODULES[stage_number]
+ stage_run_id = _start_stage_run(
+ context,
+ run_id=run_id,
+ stage_number=stage_number,
+ stage_name=stage_name,
+ attempt=attempt,
+ payload=payload,
+ )
+ try:
+ result = _execute_stage(execution_context, stage_number, payload)
+ except Exception as error:
+ _finish_stage_run(
+ context,
+ stage_run_id,
+ status="failed",
+ error_message=str(error),
+ )
+ raise
+ _finish_stage_run(
+ context,
+ stage_run_id,
+ status="succeeded",
+ output=result,
+ )
+ outputs[str(stage_number)] = result
+ _checkpoint(context, run_id, outputs)
+ with context.storage.session() as session:
+ run = session.get(PipelineRun, run_id)
+ if run is None:
+ raise RuntimeError(f"pipeline run disappeared: {run_id}")
+ run.status = "succeeded"
+ run.output_json = outputs
+ run.finished_at = utc_now()
+ run.heartbeat_at = utc_now()
+ LOGGER.info(
+ "Pipeline run succeeded",
+ extra={"event": "pipeline_succeeded", "run_id": run_id},
+ )
+ return outputs
+ except Exception as error:
+ with context.storage.session() as session:
+ run = session.get(PipelineRun, run_id)
+ if run is not None:
+ run.status = "failed"
+ run.error_message = str(error)
+ run.finished_at = utc_now()
+ run.heartbeat_at = utc_now()
+ LOGGER.exception(
+ "Pipeline run failed",
+ extra={"event": "pipeline_failed", "run_id": run_id},
+ )
+ raise
+ finally:
+ if lock_connection is not None:
+ lock_connection.execute(
+ text("SELECT pg_advisory_unlock(hashtext(:key))"),
+ {"key": f"knowledgegraph:{corpus}"},
+ )
+ lock_connection.close()
+
+
+def requeue_failed_run(context: StageContext, run_id: str) -> None:
+ with context.storage.session() as session:
+ run = session.get(PipelineRun, run_id)
+ if run is None:
+ raise ValueError(f"pipeline run does not exist: {run_id}")
+ if run.status != "failed":
+ raise ValueError("only failed pipeline runs can be requeued")
+ run.status = "queued"
+ run.error_message = None
+ run.finished_at = None
+
+
+def recover_stale_runs(context: StageContext) -> int:
+ threshold = utc_now().timestamp() - context.settings.worker_stale_run_seconds
+ cutoff = datetime.fromtimestamp(threshold, tz=timezone.utc)
+ with context.storage.session() as session:
+ result = session.execute(
+ update(PipelineRun)
+ .where(
+ PipelineRun.status == "running",
+ PipelineRun.heartbeat_at < cutoff,
+ )
+ .values(
+ status="queued",
+ error_message="Recovered after a stale worker lease",
+ finished_at=None,
+ )
+ )
+ recovered = int(result.rowcount or 0)
+ if recovered:
+ LOGGER.warning(
+ "Recovered stale pipeline runs",
+ extra={"event": "pipeline_stale_recovered", "count": recovered},
+ )
+ return recovered
+
+
+def _execute_stage(
+ context: StageContext, stage_number: int, payload: dict[str, Any]
+) -> dict[str, Any]:
+ module = load_stage(stage_number)
+ if stage_number == 1:
+ input_path = payload.get("input_jsonl")
+ return module.run(
+ context,
+ input_jsonl=Path(input_path) if input_path else None,
+ )
+ if stage_number == 3:
+ model_path = payload.get("model_file")
+ return module.run(
+ context,
+ model_file=Path(model_path) if model_path else None,
+ )
+ return module.run(context)
+
+
+def _start_stage_run(
+ context: StageContext,
+ *,
+ run_id: str,
+ stage_number: int,
+ stage_name: str,
+ attempt: int,
+ payload: dict[str, Any],
+) -> int:
+ with context.storage.session() as session:
+ stage_run = StageRun(
+ pipeline_run_id=run_id,
+ stage_number=stage_number,
+ stage_name=stage_name,
+ status="running",
+ attempt=attempt,
+ input_json=payload,
+ )
+ session.add(stage_run)
+ session.flush()
+ return stage_run.id
+
+
+def _finish_stage_run(
+ context: StageContext,
+ stage_run_id: int,
+ *,
+ status: str,
+ output: dict[str, Any] | None = None,
+ error_message: str | None = None,
+) -> None:
+ with context.storage.session() as session:
+ stage_run = session.get(StageRun, stage_run_id)
+ if stage_run is None:
+ raise RuntimeError(f"stage run disappeared: {stage_run_id}")
+ stage_run.status = status
+ stage_run.output_json = output or {}
+ stage_run.error_message = error_message
+ stage_run.finished_at = utc_now()
+
+
+def _checkpoint(
+ context: StageContext, run_id: str, outputs: dict[str, Any]
+) -> None:
+ with context.storage.session() as session:
+ run = session.get(PipelineRun, run_id)
+ if run is not None:
+ run.output_json = outputs
+ run.heartbeat_at = utc_now()
+
+
+def _successful_stage_output(
+ context: StageContext, run_id: str, stage_number: int
+) -> dict[str, Any] | None:
+ with context.storage.session() as session:
+ stage_run = session.scalar(
+ select(StageRun)
+ .where(
+ StageRun.pipeline_run_id == run_id,
+ StageRun.stage_number == stage_number,
+ StageRun.status == "succeeded",
+ )
+ .order_by(StageRun.attempt.desc())
+ .limit(1)
+ )
+ return dict(stage_run.output_json) if stage_run is not None else None
diff --git a/src/graphrag_platform/pipeline.py b/src/graphrag_platform/pipeline.py
new file mode 100644
index 0000000..685ed51
--- /dev/null
+++ b/src/graphrag_platform/pipeline.py
@@ -0,0 +1,72 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Any
+
+from graphrag_platform.config import Settings
+from graphrag_platform.llm import LLMClient
+from graphrag_platform.runtime import StageContext
+from graphrag_platform.stage_loader import load_stage
+from graphrag_platform.storage import Storage
+
+
+class KnowledgeGraphPipeline:
+ """Compatibility facade over independently executable numbered stages."""
+
+ def __init__(
+ self,
+ settings: Settings,
+ *,
+ storage: Storage | None = None,
+ llm: LLMClient | None = None,
+ ) -> None:
+ self.context = StageContext.create(settings, storage=storage, llm=llm)
+
+ @property
+ def settings(self) -> Settings:
+ return self.context.settings
+
+ @property
+ def storage(self) -> Storage:
+ return self.context.storage
+
+ def initialize(self) -> None:
+ self.context.initialize()
+
+ def stage_01_normalize(
+ self, *, input_jsonl: Path | None = None
+ ) -> dict[str, int]:
+ return load_stage(1).run(self.context, input_jsonl=input_jsonl)
+
+ def stage_02_clean_and_chunk(self) -> dict[str, int]:
+ return load_stage(2).run(self.context)
+
+ def stage_03_define_model(
+ self, model_file: Path | None = None
+ ) -> dict[str, int]:
+ return load_stage(3).run(self.context, model_file=model_file)
+
+ def stage_04_extract(self) -> dict[str, int]:
+ return load_stage(4).run(self.context)
+
+ def stage_05_reconcile(self) -> dict[str, int]:
+ return load_stage(5).run(self.context)
+
+ def stage_06_build_graph(self) -> dict[str, int]:
+ return load_stage(6).run(self.context)
+
+ def stage_07_build_communities(self) -> dict[str, int]:
+ return load_stage(7).run(self.context)
+
+ def stage_08_build_retrieval_index(self) -> dict[str, int]:
+ return load_stage(8).run(self.context)
+
+ def stage_09_retrieve(
+ self, question: str, *, top_k: int | None = None
+ ) -> dict[str, Any]:
+ return load_stage(9).run(self.context, question=question, top_k=top_k)
+
+ def stage_10_answer(
+ self, question: str, *, top_k: int | None = None
+ ) -> dict[str, Any]:
+ return load_stage(10).run(self.context, question=question, top_k=top_k)
diff --git a/src/graphrag_platform/runtime.py b/src/graphrag_platform/runtime.py
new file mode 100644
index 0000000..63e2676
--- /dev/null
+++ b/src/graphrag_platform/runtime.py
@@ -0,0 +1,50 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+from graphrag_platform.config import Settings
+from graphrag_platform.llm import LLMClient, OpenAICompatibleClient
+from graphrag_platform.storage import Storage
+
+
+@dataclass
+class StageContext:
+ settings: Settings
+ storage: Storage
+ _llm: LLMClient | None = None
+
+ @classmethod
+ def create(
+ cls,
+ settings: Settings,
+ *,
+ storage: Storage | None = None,
+ llm: LLMClient | None = None,
+ ) -> StageContext:
+ return cls(
+ settings=settings,
+ storage=storage or Storage(settings.database_url),
+ _llm=llm,
+ )
+
+ @property
+ def llm(self) -> LLMClient:
+ if self._llm is None:
+ self._llm = OpenAICompatibleClient(self.settings)
+ return self._llm
+
+ def initialize(self) -> None:
+ if self.settings.auto_create_schema:
+ self.storage.create_schema()
+ else:
+ self.storage.verify_schema()
+ (self.settings.artifact_dir / self.settings.corpus_name).mkdir(
+ parents=True, exist_ok=True
+ )
+
+ def for_corpus(self, corpus: str) -> StageContext:
+ return StageContext.create(
+ self.settings.model_copy(update={"corpus_name": corpus}),
+ storage=self.storage,
+ llm=self._llm,
+ )
diff --git a/src/graphrag_platform/sources.py b/src/graphrag_platform/sources.py
new file mode 100644
index 0000000..e5b2557
--- /dev/null
+++ b/src/graphrag_platform/sources.py
@@ -0,0 +1,106 @@
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Iterator
+from datetime import datetime
+from pathlib import Path
+from typing import Any, Protocol
+
+from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator
+from sqlalchemy import MetaData, Table, create_engine, select
+
+_TABLE_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
+
+
+class SourceRecord(BaseModel):
+ model_config = ConfigDict(extra="forbid")
+
+ schema_version: str = "1.0"
+ external_id: str = Field(min_length=1, max_length=500)
+ corpus: str | None = Field(default=None, min_length=1, max_length=200)
+ url: HttpUrl
+ body: str = Field(min_length=1)
+ content_type: str = "text/html"
+ status_code: int = Field(default=200, ge=100, le=599)
+ retrieved_at: datetime
+ metadata: dict[str, Any] = Field(default_factory=dict)
+
+ @field_validator("schema_version")
+ @classmethod
+ def require_supported_schema(cls, value: str) -> str:
+ if value != "1.0":
+ raise ValueError(f"unsupported source schema version: {value}")
+ return value
+
+
+class SourceReader(Protocol):
+ def records(self) -> Iterator[SourceRecord]: ...
+
+
+class JsonLinesSourceReader:
+ def __init__(self, path: Path) -> None:
+ self.path = path
+
+ def records(self) -> Iterator[SourceRecord]:
+ with self.path.open("r", encoding="utf-8") as stream:
+ for line_number, line in enumerate(stream, start=1):
+ if not line.strip():
+ continue
+ try:
+ payload = json.loads(line)
+ yield SourceRecord.model_validate(payload)
+ except (json.JSONDecodeError, ValueError) as error:
+ raise ValueError(
+ f"{self.path}:{line_number}: invalid source record: {error}"
+ ) from error
+
+
+class PostgreSqlSourceReader:
+ def __init__(self, database_url: str, table_name: str) -> None:
+ if not database_url.startswith("postgresql"):
+ raise ValueError("SOURCE_DATABASE_URL must use PostgreSQL")
+ if not _TABLE_NAME_RE.fullmatch(table_name):
+ raise ValueError("SOURCE_TABLE_NAME must be an unqualified SQL identifier")
+ self.engine = create_engine(database_url, pool_pre_ping=True)
+ self.table_name = table_name
+
+ def records(self) -> Iterator[SourceRecord]:
+ metadata = MetaData()
+ table = Table(self.table_name, metadata, autoload_with=self.engine)
+ required = {
+ "schema_version",
+ "external_id",
+ "url",
+ "body",
+ "retrieved_at",
+ }
+ missing = required - set(table.columns.keys())
+ if missing:
+ raise ValueError(
+ f"source table {self.table_name} is missing columns: "
+ f"{', '.join(sorted(missing))}"
+ )
+ with self.engine.connect() as connection:
+ for row in connection.execute(select(table).order_by(table.c.external_id)):
+ payload = {
+ key: value
+ for key, value in row._mapping.items()
+ if key in SourceRecord.model_fields
+ }
+ yield SourceRecord.model_validate(payload)
+
+
+def source_reader(
+ *,
+ input_jsonl: Path | None,
+ source_database_url: str | None,
+ source_table_name: str,
+) -> SourceReader:
+ if input_jsonl is not None and source_database_url:
+ raise ValueError("provide either --input-jsonl or SOURCE_DATABASE_URL, not both")
+ if input_jsonl is not None:
+ return JsonLinesSourceReader(input_jsonl)
+ if source_database_url:
+ return PostgreSqlSourceReader(source_database_url, source_table_name)
+ raise ValueError("provide --input-jsonl or configure SOURCE_DATABASE_URL")
diff --git a/src/graphrag_platform/stage_loader.py b/src/graphrag_platform/stage_loader.py
new file mode 100644
index 0000000..4a0a8c0
--- /dev/null
+++ b/src/graphrag_platform/stage_loader.py
@@ -0,0 +1,25 @@
+from __future__ import annotations
+
+from importlib import import_module
+from types import ModuleType
+
+STAGE_MODULES = {
+ 1: "01_normalize_content",
+ 2: "02_clean_content",
+ 3: "03_define_knowledge_model",
+ 4: "04_extract_knowledge",
+ 5: "05_reconcile_knowledge",
+ 6: "06_build_knowledge_graph",
+ 7: "07_build_communities",
+ 8: "08_build_retrieval_index",
+ 9: "09_query_graphrag",
+ 10: "10_generate_answer",
+}
+
+
+def load_stage(stage: int) -> ModuleType:
+ try:
+ module_name = STAGE_MODULES[stage]
+ except KeyError as error:
+ raise ValueError(f"unsupported stage {stage}") from error
+ return import_module(module_name)
diff --git a/src/graphrag_platform/storage.py b/src/graphrag_platform/storage.py
new file mode 100644
index 0000000..b0d7e8e
--- /dev/null
+++ b/src/graphrag_platform/storage.py
@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+from collections.abc import Iterator
+from contextlib import contextmanager
+from typing import Any, TypeVar
+
+from sqlalchemy import create_engine, delete, event, inspect, select, text
+from sqlalchemy.engine import Engine
+from sqlalchemy.orm import Session
+
+from graphrag_platform.models import (
+ Base,
+ Chunk,
+ Community,
+ Document,
+ Edge,
+ Extraction,
+ KnowledgeModel,
+ Node,
+ RawDocument,
+ ResolvedEntity,
+ ResolvedRelationship,
+ SearchItem,
+)
+
+ModelT = TypeVar("ModelT", bound=Base)
+
+
+class Storage:
+ def __init__(self, database_url: str) -> None:
+ connect_args = {"check_same_thread": False} if database_url.startswith("sqlite") else {}
+ self.engine: Engine = create_engine(
+ database_url,
+ future=True,
+ pool_pre_ping=True,
+ connect_args=connect_args,
+ )
+ if database_url.startswith("sqlite"):
+ event.listen(self.engine, "connect", self._enable_sqlite_foreign_keys)
+
+ @staticmethod
+ def _enable_sqlite_foreign_keys(
+ dbapi_connection: Any, connection_record: Any
+ ) -> None:
+ del connection_record
+ cursor = dbapi_connection.cursor()
+ cursor.execute("PRAGMA foreign_keys=ON")
+ cursor.close()
+
+ def create_schema(self) -> None:
+ if self.engine.dialect.name == "postgresql":
+ with self.engine.begin() as connection:
+ connection.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
+ Base.metadata.create_all(self.engine)
+
+ def verify_schema(self) -> None:
+ tables = set(inspect(self.engine).get_table_names())
+ required = {"pipeline_runs", "stage_runs", "documents", "retrieval_items"}
+ missing = required - tables
+ if missing:
+ raise RuntimeError(
+ "database migrations are required; missing tables: "
+ f"{', '.join(sorted(missing))}"
+ )
+ if self.engine.dialect.name == "postgresql":
+ with self.engine.connect() as connection:
+ extension = connection.scalar(
+ text(
+ "SELECT extname FROM pg_extension "
+ "WHERE extname = 'vector'"
+ )
+ )
+ if extension != "vector":
+ raise RuntimeError("database migrations are required: vector extension")
+
+ @property
+ def supports_vector_search(self) -> bool:
+ return self.engine.dialect.name == "postgresql"
+
+ def ensure_vector_index(self, dimensions: int) -> None:
+ if not self.supports_vector_search:
+ return
+ index_name = f"ix_retrieval_items_embedding_hnsw_{dimensions}"
+ statement = f"""
+ CREATE INDEX IF NOT EXISTS {index_name}
+ ON retrieval_items
+ USING hnsw ((embedding::vector({dimensions})) vector_cosine_ops)
+ WHERE vector_dims(embedding) = {dimensions}
+ """
+ with self.engine.begin() as connection:
+ connection.execute(text(statement))
+
+ @contextmanager
+ def session(self) -> Iterator[Session]:
+ with Session(self.engine) as session:
+ try:
+ yield session
+ session.commit()
+ except Exception:
+ session.rollback()
+ raise
+
+ def upsert(
+ self,
+ session: Session,
+ model: type[ModelT],
+ identity: dict[str, Any],
+ values: dict[str, Any],
+ ) -> ModelT:
+ instance = session.scalar(select(model).filter_by(**identity))
+ if instance is None:
+ instance = model(**identity, **values)
+ session.add(instance)
+ else:
+ for key, value in values.items():
+ setattr(instance, key, value)
+ session.flush()
+ return instance
+
+ def clear_from_stage(self, corpus: str, stage: int) -> None:
+ stage_tables: list[tuple[int, type[Base]]] = [
+ (8, SearchItem),
+ (7, Community),
+ (6, Edge),
+ (6, Node),
+ (5, ResolvedRelationship),
+ (5, ResolvedEntity),
+ (4, Extraction),
+ (2, Chunk),
+ (1, RawDocument),
+ (1, Document),
+ ]
+ with self.session() as session:
+ for table_stage, model in stage_tables:
+ if table_stage >= stage:
+ session.execute(delete(model).where(model.corpus == corpus))
+ if stage <= 3:
+ session.execute(
+ delete(KnowledgeModel).where(KnowledgeModel.corpus == corpus)
+ )
\ No newline at end of file
diff --git a/src/graphrag_platform/worker.py b/src/graphrag_platform/worker.py
new file mode 100644
index 0000000..cdb5e10
--- /dev/null
+++ b/src/graphrag_platform/worker.py
@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+import argparse
+import logging
+import time
+
+from graphrag_platform.config import Settings
+from graphrag_platform.observability import configure_logging
+from graphrag_platform.orchestrator import (
+ RunDeferredError,
+ claim_next_run,
+ execute_run,
+ recover_stale_runs,
+)
+from graphrag_platform.runtime import StageContext
+
+LOGGER = logging.getLogger(__name__)
+
+
+def work_once(context: StageContext) -> bool:
+ run_id = claim_next_run(context)
+ if run_id is None:
+ return False
+ try:
+ execute_run(context, run_id)
+ except RunDeferredError:
+ LOGGER.info(
+ "Worker deferred a run for an active corpus",
+ extra={"event": "worker_job_deferred", "run_id": run_id},
+ )
+ except Exception:
+ LOGGER.exception(
+ "Worker execution failed",
+ extra={"event": "worker_job_failed", "run_id": run_id},
+ )
+ return True
+
+
+def run_worker(context: StageContext, *, once: bool = False) -> None:
+ context.initialize()
+ recover_stale_runs(context)
+ while True:
+ worked = work_once(context)
+ if once:
+ return
+ if not worked:
+ time.sleep(context.settings.worker_poll_seconds)
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description="Run the KnowledgeGraph job worker.")
+ parser.add_argument("--env-file", default=".env")
+ parser.add_argument("--once", action="store_true")
+ args = parser.parse_args()
+ settings = Settings(_env_file=args.env_file)
+ configure_logging(settings.log_level)
+ run_worker(StageContext.create(settings), once=args.once)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/src/templates/knowledge_model.json b/src/templates/knowledge_model.json
new file mode 100644
index 0000000..81209cb
--- /dev/null
+++ b/src/templates/knowledge_model.json
@@ -0,0 +1,33 @@
+{
+ "entity_types": [
+ "Person",
+ "Organization",
+ "Product",
+ "Concept",
+ "Event",
+ "Location"
+ ],
+ "relationship_types": [
+ "AFFILIATED_WITH",
+ "CREATED",
+ "DEPENDS_ON",
+ "MENTIONS",
+ "PART_OF",
+ "RELATED_TO"
+ ],
+ "node_template": {
+ "canonical_name": "string",
+ "entity_type": "one configured entity type",
+ "description": "source-grounded string",
+ "aliases": [
+ "string"
+ ]
+ },
+ "edge_template": {
+ "source": "entity name",
+ "target": "entity name",
+ "relationship_type": "one configured relationship type",
+ "description": "source-grounded string"
+ },
+ "extraction_instructions": "Extract only entities and relationships explicitly supported by the supplied text."
+}
diff --git a/src/templates/mcp_documents.jsonl b/src/templates/mcp_documents.jsonl
new file mode 100644
index 0000000..0d5ff8f
--- /dev/null
+++ b/src/templates/mcp_documents.jsonl
@@ -0,0 +1 @@
+{"schema_version":"1.0","external_id":"docs-example-article","corpus":"product-knowledge","url":"https://docs.example.com/article","content_type":"text/html","status_code":200,"retrieved_at":"2026-01-01T00:00:00Z","body":"Article titleArticle title
Article text collected with Web-Access.
","metadata":{"collector":"Web-Access","stored_by":"Data-Store"}}
diff --git a/tests/conftest.py b/tests/conftest.py
new file mode 100644
index 0000000..c3066e7
--- /dev/null
+++ b/tests/conftest.py
@@ -0,0 +1,58 @@
+from __future__ import annotations
+
+import hashlib
+from collections.abc import Sequence
+from typing import Any
+
+
+class FakeLLM:
+ chat_model = "fake-chat-model"
+ embedding_model = "fake-embedding-model"
+
+ def complete_json(self, system: str, user: str) -> dict[str, Any]:
+ if "community" in system.casefold():
+ return {
+ "title": "Atlas product community",
+ "summary": "Acme created Atlas, a graph retrieval product.",
+ }
+ return {
+ "entities": [
+ {
+ "canonical_name": "Acme",
+ "entity_type": "Organization",
+ "description": "Acme is a technology organization.",
+ "aliases": ["Acme Corp"],
+ },
+ {
+ "canonical_name": "Atlas",
+ "entity_type": "Product",
+ "description": "Atlas is a graph retrieval product.",
+ "aliases": [],
+ },
+ ],
+ "relationships": [
+ {
+ "source": "Acme",
+ "target": "Atlas",
+ "relationship_type": "CREATED",
+ "description": "Acme created Atlas.",
+ }
+ ],
+ }
+
+ def complete_text(self, system: str, user: str) -> str:
+ return "Acme created Atlas. [S1]"
+
+ def embed(self, texts: Sequence[str]) -> list[list[float]]:
+ vectors: list[list[float]] = []
+ for text in texts:
+ normalized = text.casefold()
+ digest = hashlib.sha256(normalized.encode("utf-8")).digest()
+ vectors.append(
+ [
+ float("atlas" in normalized),
+ float("acme" in normalized),
+ digest[0] / 255,
+ ]
+ )
+ return vectors
diff --git a/tests/fixtures/external_documents.jsonl b/tests/fixtures/external_documents.jsonl
new file mode 100644
index 0000000..134ff86
--- /dev/null
+++ b/tests/fixtures/external_documents.jsonl
@@ -0,0 +1,3 @@
+{"schema_version":"1.0","external_id":"atlas-overview","corpus":"acceptance-corpus","url":"https://docs.example/atlas","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:00:00Z","body":"Atlas OverviewAtlas
Acme Corporation created Atlas. Atlas provides graph retrieval for knowledge applications.
","metadata":{"collector":"Web-Access","stored_by":"Data-Store","tags":["product","graph"]}}
+{"schema_version":"1.0","external_id":"atlas-architecture","corpus":"acceptance-corpus","url":"https://docs.example/atlas/architecture","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:01:00Z","body":"Atlas ArchitectureArchitecture
Acme Corp designed Atlas to combine a knowledge graph with vector retrieval. Atlas depends on PostgreSQL and PgVector.
","metadata":{"collector":"Web-Access","stored_by":"Data-Store","description":"Contains commas, \"quotes\", and aliases."}}
+{"schema_version":"1.0","external_id":"atlas-overview-copy","corpus":"acceptance-corpus","url":"https://mirror.example/atlas","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:02:00Z","body":"Atlas OverviewAtlas
Acme Corporation created Atlas. Atlas provides graph retrieval for knowledge applications.
","metadata":{"collector":"Web-Access","stored_by":"Data-Store","duplicate_candidate":true}}
diff --git a/tests/integration/test_api_worker.py b/tests/integration/test_api_worker.py
new file mode 100644
index 0000000..c1b6424
--- /dev/null
+++ b/tests/integration/test_api_worker.py
@@ -0,0 +1,284 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime, timedelta, timezone
+from pathlib import Path
+from uuid import uuid4
+
+import jwt
+from fastapi.testclient import TestClient
+
+from graphrag_platform.api import create_app
+from graphrag_platform.auth import Permission
+from graphrag_platform.config import Settings
+from graphrag_platform.runtime import StageContext
+from graphrag_platform.worker import work_once
+
+from conftest import FakeLLM
+
+
+def _token(settings: Settings, mask: int) -> str:
+ now = datetime.now(timezone.utc)
+ return jwt.encode(
+ {
+ "iss": settings.internal_jwt_issuer,
+ "aud": settings.internal_jwt_audience,
+ "sub": "dev-test-client",
+ "iat": now,
+ "exp": now + timedelta(minutes=10),
+ "jti": str(uuid4()),
+ "features": {settings.internal_feature_name: mask},
+ },
+ settings.internal_jwt_secret,
+ algorithm="HS256",
+ )
+
+
+def test_bff_contract_and_worker_execute_durable_job(tmp_path: Path) -> None:
+ source = tmp_path / "source.jsonl"
+ source.write_text(
+ json.dumps(
+ {
+ "schema_version": "1.0",
+ "external_id": "atlas",
+ "corpus": "api-corpus",
+ "url": "https://docs.example/atlas",
+ "retrieved_at": "2026-10-13T18:00:00Z",
+ "body": (
+ "Atlas"
+ "Acme created Atlas for graph retrieval."
+ ),
+ }
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+ settings = Settings(
+ _env_file=None,
+ database_url=f"sqlite:///{tmp_path / 'api.db'}",
+ auto_create_schema=True,
+ artifact_dir=tmp_path / "artifacts",
+ llm_embedding_dimensions=3,
+ internal_jwt_secret="dev-test-secret-at-least-32-bytes-long",
+ )
+ app = create_app(settings, llm=FakeLLM())
+ full_token = _token(
+ settings,
+ int(Permission.READ | Permission.ADD | Permission.EXECUTE),
+ )
+
+ with TestClient(app) as client:
+ assert client.get("/internal/knowledge-graph/health").status_code == 200
+ assert (
+ client.post(
+ "/internal/knowledge-graph/jobs",
+ json={
+ "corpus": "api-corpus",
+ "from_stage": 1,
+ "through_stage": 8,
+ "input_jsonl": str(source),
+ },
+ ).status_code
+ == 401
+ )
+ execute_only = _token(settings, int(Permission.EXECUTE))
+ assert (
+ client.post(
+ "/internal/knowledge-graph/jobs",
+ headers={"Authorization": f"Bearer {execute_only}"},
+ json={
+ "corpus": "api-corpus",
+ "from_stage": 1,
+ "through_stage": 8,
+ "input_jsonl": str(source),
+ },
+ ).status_code
+ == 403
+ )
+ queued = client.post(
+ "/internal/knowledge-graph/jobs",
+ headers={"Authorization": f"Bearer {full_token}"},
+ json={
+ "corpus": "api-corpus",
+ "from_stage": 1,
+ "through_stage": 8,
+ "input_jsonl": str(source),
+ },
+ )
+ assert queued.status_code == 202
+ job_id = queued.json()["job_id"]
+
+ assert work_once(StageContext.create(settings, llm=FakeLLM()))
+
+ status_response = client.get(
+ f"/internal/knowledge-graph/jobs/{job_id}",
+ headers={"Authorization": f"Bearer {full_token}"},
+ )
+ assert status_response.status_code == 200
+ assert status_response.json()["status"] == "succeeded"
+ assert len(status_response.json()["stages"]) == 8
+
+ answer = client.post(
+ "/internal/knowledge-graph/answer",
+ headers={"Authorization": f"Bearer {full_token}"},
+ json={"corpus": "api-corpus", "question": "Who created Atlas?"},
+ )
+ assert answer.status_code == 200
+ assert answer.json()["answer"] == "Acme created Atlas. [S1]"
+
+
+def test_failed_job_can_be_repaired_and_retried(tmp_path: Path) -> None:
+ source = tmp_path / "retry-source.jsonl"
+ source.write_text(
+ json.dumps(
+ {
+ "schema_version": "2.0",
+ "external_id": "retry-record",
+ "corpus": "retry-corpus",
+ "url": "https://docs.example/retry",
+ "retrieved_at": "2026-10-13T18:00:00Z",
+ "body": "invalid schema",
+ }
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+ settings = Settings(
+ _env_file=None,
+ database_url=f"sqlite:///{tmp_path / 'retry.db'}",
+ auto_create_schema=True,
+ artifact_dir=tmp_path / "artifacts",
+ llm_embedding_dimensions=3,
+ internal_jwt_secret="dev-test-secret-at-least-32-bytes-long",
+ )
+ app = create_app(settings, llm=FakeLLM())
+ token = _token(
+ settings,
+ int(Permission.READ | Permission.ADD | Permission.EXECUTE),
+ )
+ headers = {"Authorization": f"Bearer {token}"}
+
+ with TestClient(app) as client:
+ queued = client.post(
+ "/internal/knowledge-graph/jobs",
+ headers=headers,
+ json={
+ "corpus": "retry-corpus",
+ "from_stage": 1,
+ "through_stage": 1,
+ "input_jsonl": str(source),
+ },
+ )
+ job_id = queued.json()["job_id"]
+ assert work_once(StageContext.create(settings, llm=FakeLLM()))
+ assert (
+ client.get(
+ f"/internal/knowledge-graph/jobs/{job_id}",
+ headers=headers,
+ ).json()["status"]
+ == "failed"
+ )
+
+ source.write_text(
+ json.dumps(
+ {
+ "schema_version": "1.0",
+ "external_id": "retry-record",
+ "corpus": "retry-corpus",
+ "url": "https://docs.example/retry",
+ "retrieved_at": "2026-10-13T18:00:00Z",
+ "content_type": "text/plain",
+ "body": "Corrected source content.",
+ }
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+ retried = client.post(
+ f"/internal/knowledge-graph/jobs/{job_id}/retry",
+ headers=headers,
+ )
+ assert retried.status_code == 202
+ assert work_once(StageContext.create(settings, llm=FakeLLM()))
+ status_response = client.get(
+ f"/internal/knowledge-graph/jobs/{job_id}",
+ headers=headers,
+ )
+ assert status_response.json()["status"] == "succeeded"
+ assert status_response.json()["attempt"] == 2
+
+
+def test_retry_resumes_after_last_successful_stage(tmp_path: Path) -> None:
+ source = tmp_path / "resume-source.jsonl"
+ source.write_text(
+ json.dumps(
+ {
+ "schema_version": "1.0",
+ "external_id": "resume-record",
+ "corpus": "resume-corpus",
+ "url": "https://docs.example/resume",
+ "retrieved_at": "2026-10-13T18:00:00Z",
+ "content_type": "text/plain",
+ "body": "Acme created Atlas for graph retrieval.",
+ }
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+ model_file = tmp_path / "model.json"
+ model_file.write_text("[]", encoding="utf-8")
+ settings = Settings(
+ _env_file=None,
+ database_url=f"sqlite:///{tmp_path / 'resume.db'}",
+ auto_create_schema=True,
+ artifact_dir=tmp_path / "artifacts",
+ llm_embedding_dimensions=3,
+ internal_jwt_secret="dev-test-secret-at-least-32-bytes-long",
+ )
+ app = create_app(settings, llm=FakeLLM())
+ token = _token(
+ settings,
+ int(Permission.READ | Permission.ADD | Permission.EXECUTE),
+ )
+ headers = {"Authorization": f"Bearer {token}"}
+
+ with TestClient(app) as client:
+ queued = client.post(
+ "/internal/knowledge-graph/jobs",
+ headers=headers,
+ json={
+ "corpus": "resume-corpus",
+ "from_stage": 1,
+ "through_stage": 3,
+ "input_jsonl": str(source),
+ "model_file": str(model_file),
+ },
+ )
+ job_id = queued.json()["job_id"]
+ assert work_once(StageContext.create(settings, llm=FakeLLM()))
+ failed = client.get(
+ f"/internal/knowledge-graph/jobs/{job_id}", headers=headers
+ ).json()
+ assert failed["status"] == "failed"
+
+ model_file.write_text(
+ json.dumps(settings.model_definition()),
+ encoding="utf-8",
+ )
+ assert (
+ client.post(
+ f"/internal/knowledge-graph/jobs/{job_id}/retry",
+ headers=headers,
+ ).status_code
+ == 202
+ )
+ assert work_once(StageContext.create(settings, llm=FakeLLM()))
+ succeeded = client.get(
+ f"/internal/knowledge-graph/jobs/{job_id}", headers=headers
+ ).json()
+
+ assert succeeded["status"] == "succeeded"
+ stage_numbers = [row["stage"] for row in succeeded["stages"]]
+ assert stage_numbers.count(1) == 1
+ assert stage_numbers.count(2) == 1
+ assert stage_numbers.count(3) == 2
diff --git a/tests/integration/test_full_pipeline.py b/tests/integration/test_full_pipeline.py
new file mode 100644
index 0000000..6cb007b
--- /dev/null
+++ b/tests/integration/test_full_pipeline.py
@@ -0,0 +1,69 @@
+import json
+from pathlib import Path
+
+from sqlalchemy import func, select
+
+from graphrag_platform.config import Settings
+from graphrag_platform.models import Community, Edge, Node, SearchItem
+from graphrag_platform.runtime import StageContext
+from graphrag_platform.stage_loader import load_stage
+
+from conftest import FakeLLM
+
+
+def test_all_ten_stages_build_and_query_graph(tmp_path: Path) -> None:
+ input_file = tmp_path / "documents.jsonl"
+ input_file.write_text(
+ json.dumps(
+ {
+ "schema_version": "1.0",
+ "external_id": "atlas-doc",
+ "corpus": "atlas-corpus",
+ "url": "https://docs.example/atlas",
+ "retrieved_at": "2026-10-13T18:00:00Z",
+ "body": (
+ "Atlas"
+ "Atlas
Acme created Atlas. Atlas provides graph "
+ "retrieval for knowledge applications.
"
+ ),
+ "metadata": {"collector": "Web-Access", "stored_by": "Data-Store"},
+ }
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+ settings = Settings(
+ _env_file=None,
+ database_url=f"sqlite:///{tmp_path / 'graph.db'}",
+ auto_create_schema=True,
+ artifact_dir=tmp_path / "artifacts",
+ corpus_name="atlas-corpus",
+ chunk_size=300,
+ chunk_overlap=30,
+ retrieval_top_k=5,
+ llm_embedding_dimensions=3,
+ )
+ context = StageContext.create(settings, llm=FakeLLM())
+
+ assert load_stage(1).run(context, input_jsonl=input_file)["normalized"] == 1
+ assert load_stage(2).run(context)["chunks"] == 1
+ assert load_stage(3).run(context)["entity_types"] == 6
+ assert load_stage(4).run(context)["extracted_chunks"] == 1
+ assert load_stage(5).run(context) == {"entities": 2, "relationships": 1}
+ assert load_stage(6).run(context) == {"nodes": 2, "edges": 1}
+ assert load_stage(7).run(context)["communities"] == 1
+ assert load_stage(8).run(context)["indexed_items"] == 5
+
+ retrieval = load_stage(9).run(context, question="Who created Atlas?")
+ response = load_stage(10).run(context, question="Who created Atlas?")
+
+ assert retrieval["context"]
+ assert retrieval["graph_context"][0]["relationship"] == "CREATED"
+ assert retrieval["sources"][0]["url"] == "https://docs.example/atlas"
+ assert response["answer"] == "Acme created Atlas. [S1]"
+ assert response["sources"][0]["citation"] == "S1"
+ with context.storage.session() as session:
+ assert session.scalar(select(func.count()).select_from(Node)) == 2
+ assert session.scalar(select(func.count()).select_from(Edge)) == 1
+ assert session.scalar(select(func.count()).select_from(Community)) == 1
+ assert session.scalar(select(func.count()).select_from(SearchItem)) == 5
diff --git a/tests/integration/test_pgvector_postgres.py b/tests/integration/test_pgvector_postgres.py
new file mode 100644
index 0000000..33ede97
--- /dev/null
+++ b/tests/integration/test_pgvector_postgres.py
@@ -0,0 +1,140 @@
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import pytest
+from sqlalchemy import text
+
+from graphrag_platform.config import Settings
+from graphrag_platform.models import PipelineRun
+from graphrag_platform.orchestrator import (
+ RunDeferredError,
+ claim_next_run,
+ execute_run,
+ submit_run,
+)
+from graphrag_platform.runtime import StageContext
+from graphrag_platform.stage_loader import load_stage
+
+from conftest import FakeLLM
+
+POSTGRES_URL = os.getenv("TEST_POSTGRES_URL")
+
+
+@pytest.mark.skipif(
+ not POSTGRES_URL,
+ reason="TEST_POSTGRES_URL is required for the PostgreSQL integration test",
+)
+def test_pgvector_index_and_graph_retrieval(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ settings = Settings(
+ _env_file=None,
+ database_url=POSTGRES_URL,
+ source_database_url=POSTGRES_URL,
+ source_table_name="external_documents",
+ artifact_dir=tmp_path / "artifacts",
+ corpus_name="pgvector-integration",
+ chunk_size=300,
+ chunk_overlap=30,
+ retrieval_top_k=5,
+ llm_embedding_dimensions=3,
+ )
+ monkeypatch.setenv("DATABASE_URL", POSTGRES_URL)
+ from alembic import command
+ from alembic.config import Config
+
+ alembic_config = Config("alembic.ini")
+ command.downgrade(alembic_config, "base")
+ command.upgrade(alembic_config, "head")
+ context = StageContext.create(settings, llm=FakeLLM())
+ context.storage.clear_from_stage(settings.corpus_name, 1)
+ with context.storage.engine.begin() as connection:
+ connection.execute(text("DROP TABLE IF EXISTS external_documents"))
+ connection.execute(
+ text(
+ """
+ CREATE TABLE external_documents (
+ id bigserial PRIMARY KEY,
+ external_id text NOT NULL UNIQUE,
+ schema_version text NOT NULL,
+ corpus text,
+ url text NOT NULL,
+ body text NOT NULL,
+ content_type text NOT NULL,
+ status_code integer NOT NULL,
+ retrieved_at timestamptz NOT NULL,
+ metadata jsonb NOT NULL,
+ created_at timestamptz NOT NULL DEFAULT now()
+ )
+ """
+ )
+ )
+ connection.execute(
+ text(
+ """
+ INSERT INTO external_documents (
+ external_id, schema_version, corpus, url, body,
+ content_type, status_code, retrieved_at, metadata
+ ) VALUES (
+ 'atlas-doc', '1.0', 'pgvector-integration',
+ 'https://docs.example/atlas',
+ 'AtlasAtlas
Acme created Atlas. Atlas provides graph retrieval for knowledge applications.
',
+ 'text/html', 200, '2026-10-13T18:00:00Z',
+ '{"collector":"Web-Access","stored_by":"Data-Store"}'::jsonb
+ )
+ """
+ )
+ )
+
+ load_stage(1).run(context)
+ load_stage(2).run(context)
+ load_stage(3).run(context)
+ load_stage(4).run(context)
+ load_stage(5).run(context)
+ load_stage(6).run(context)
+ load_stage(7).run(context)
+ load_stage(8).run(context)
+ result = load_stage(9).run(context, question="Who created Atlas?")
+
+ assert result["context"]
+ assert result["graph_context"][0]["relationship"] == "CREATED"
+ assert result["sources"][0]["url"] == "https://docs.example/atlas"
+ with context.storage.engine.connect() as connection:
+ assert connection.scalar(
+ text("SELECT extversion FROM pg_extension WHERE extname = 'vector'")
+ )
+ index_definition = connection.scalar(
+ text(
+ "SELECT indexdef FROM pg_indexes "
+ "WHERE indexname = 'ix_retrieval_items_embedding_hnsw_3'"
+ )
+ )
+ assert index_definition is not None
+ assert "USING hnsw" in index_definition
+
+ run_id = submit_run(
+ context,
+ from_stage=2,
+ through_stage=2,
+ payload={},
+ requested_by="integration-test",
+ correlation_id="pg-lock-test",
+ permission_mask=31,
+ )
+ with context.storage.engine.connect() as lock_connection:
+ assert lock_connection.scalar(
+ text("SELECT pg_try_advisory_lock(hashtext(:key))"),
+ {"key": "knowledgegraph:pgvector-integration"},
+ )
+ with pytest.raises(RunDeferredError):
+ execute_run(context, run_id)
+ lock_connection.execute(
+ text("SELECT pg_advisory_unlock(hashtext(:key))"),
+ {"key": "knowledgegraph:pgvector-integration"},
+ )
+ with context.storage.session() as session:
+ assert session.get(PipelineRun, run_id).status == "queued"
+ assert claim_next_run(context) == run_id
+ assert execute_run(context, run_id)["2"]["chunks"] == 1
diff --git a/tests/performance/test_vector_query_load.py b/tests/performance/test_vector_query_load.py
new file mode 100644
index 0000000..8b4d20c
--- /dev/null
+++ b/tests/performance/test_vector_query_load.py
@@ -0,0 +1,64 @@
+from __future__ import annotations
+
+import os
+import time
+
+import pytest
+
+from graphrag_platform.config import Settings
+from graphrag_platform.models import SearchItem
+from graphrag_platform.runtime import StageContext
+from graphrag_platform.stage_loader import load_stage
+
+from conftest import FakeLLM
+
+POSTGRES_URL = os.getenv("TEST_POSTGRES_URL")
+RUN_PERFORMANCE_TESTS = os.getenv("RUN_PERFORMANCE_TESTS") == "1"
+
+
+@pytest.mark.performance
+@pytest.mark.skipif(
+ not POSTGRES_URL or not RUN_PERFORMANCE_TESTS,
+ reason="Set TEST_POSTGRES_URL and RUN_PERFORMANCE_TESTS=1",
+)
+def test_vector_query_latency_at_configured_load() -> None:
+ item_count = int(os.getenv("PERF_VECTOR_ITEM_COUNT", "10000"))
+ maximum_seconds = float(os.getenv("PERF_MAX_QUERY_SECONDS", "2.0"))
+ settings = Settings(
+ _env_file=None,
+ database_url=POSTGRES_URL,
+ corpus_name="performance-corpus",
+ llm_embedding_dimensions=3,
+ retrieval_top_k=10,
+ )
+ context = StageContext.create(settings, llm=FakeLLM())
+ context.initialize()
+ with context.storage.session() as session:
+ session.query(SearchItem).filter_by(corpus=settings.corpus_name).delete()
+ session.add_all(
+ SearchItem(
+ corpus=settings.corpus_name,
+ item_type="benchmark",
+ item_key=str(index),
+ text=f"Benchmark graph item {index}",
+ source_url=None,
+ metadata_json={},
+ embedding=[
+ float(index % 2),
+ float((index // 2) % 2),
+ (index % 255) / 255,
+ ],
+ embedding_model=FakeLLM.embedding_model,
+ )
+ for index in range(item_count)
+ )
+ context.storage.ensure_vector_index(3)
+
+ started = time.perf_counter()
+ result = load_stage(9).run(context, question="Atlas graph retrieval", top_k=10)
+ elapsed = time.perf_counter() - started
+
+ assert len(result["context"]) == 10
+ assert elapsed <= maximum_seconds, (
+ f"vector query took {elapsed:.3f}s; threshold is {maximum_seconds:.3f}s"
+ )
diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py
new file mode 100644
index 0000000..4546db8
--- /dev/null
+++ b/tests/unit/test_config.py
@@ -0,0 +1,25 @@
+from pathlib import Path
+
+from graphrag_platform.config import Settings
+
+
+def test_settings_parse_runtime_templates(monkeypatch) -> None:
+ monkeypatch.setenv("ENTITY_TYPES", "Company,Product")
+ monkeypatch.setenv("RELATIONSHIP_TYPES", "CREATED,DEPENDS_ON")
+ monkeypatch.setenv("NODE_TEMPLATE", '{"name":"string"}')
+
+ settings = Settings(_env_file=None)
+
+ assert settings.entity_types == ["Company", "Product"]
+ assert settings.relationship_types == ["CREATED", "DEPENDS_ON"]
+ assert settings.node_template == {"name": "string"}
+ assert settings.database_url.startswith("postgresql+psycopg://")
+
+
+def test_sqlite_is_available_as_explicit_local_override(tmp_path: Path) -> None:
+ settings = Settings(
+ _env_file=None,
+ database_url=f"sqlite:///{tmp_path / 'graph.db'}",
+ )
+
+ assert settings.database_url.startswith("sqlite:///")
diff --git a/tests/unit/test_llm.py b/tests/unit/test_llm.py
new file mode 100644
index 0000000..8933b05
--- /dev/null
+++ b/tests/unit/test_llm.py
@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+import httpx
+
+from graphrag_platform.config import Settings
+from graphrag_platform.llm import OpenAICompatibleClient
+
+
+def test_llm_retries_rate_limit_and_honors_retry_after() -> None:
+ requests = 0
+ delays: list[float] = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ nonlocal requests
+ requests += 1
+ if requests == 1:
+ return httpx.Response(
+ 429,
+ headers={"Retry-After": "0.25"},
+ json={"error": "rate limited"},
+ request=request,
+ )
+ return httpx.Response(
+ 200,
+ json={"choices": [{"message": {"content": "answer"}}]},
+ request=request,
+ )
+
+ settings = Settings(
+ _env_file=None,
+ llm_api_key="test-key",
+ llm_max_retries=2,
+ )
+ client = OpenAICompatibleClient(
+ settings,
+ transport=httpx.MockTransport(handler),
+ sleeper=delays.append,
+ jitter=lambda: 0,
+ )
+
+ assert client.complete_text("system", "user") == "answer"
+ assert requests == 2
+ assert delays == [0.25]
+
+
+def test_llm_does_not_retry_authentication_failure() -> None:
+ requests = 0
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ nonlocal requests
+ requests += 1
+ return httpx.Response(401, json={"error": "unauthorized"}, request=request)
+
+ client = OpenAICompatibleClient(
+ Settings(_env_file=None, llm_api_key="bad-key", llm_max_retries=3),
+ transport=httpx.MockTransport(handler),
+ sleeper=lambda _: None,
+ )
+
+ try:
+ client.complete_text("system", "user")
+ except httpx.HTTPStatusError:
+ pass
+ else:
+ raise AssertionError("Expected an HTTPStatusError")
+ assert requests == 1
diff --git a/tests/unit/test_pipeline.py b/tests/unit/test_pipeline.py
new file mode 100644
index 0000000..c77d029
--- /dev/null
+++ b/tests/unit/test_pipeline.py
@@ -0,0 +1,113 @@
+from pathlib import Path
+
+import pytest
+from sqlalchemy import select
+
+from graphrag_platform.config import Settings
+from graphrag_platform.models import Chunk, Document, Extraction, ResolvedRelationship
+from graphrag_platform.runtime import StageContext
+from graphrag_platform.stage_loader import load_stage
+
+from conftest import FakeLLM
+
+
+def test_chunk_overlap_must_be_smaller_than_chunk_size(tmp_path: Path) -> None:
+ settings = Settings(
+ _env_file=None,
+ database_url=f"sqlite:///{tmp_path / 'graph.db'}",
+ auto_create_schema=True,
+ artifact_dir=tmp_path / "artifacts",
+ chunk_size=200,
+ chunk_overlap=200,
+ )
+ stage = load_stage(2)
+
+ with pytest.raises(ValueError, match="CHUNK_OVERLAP"):
+ stage.chunk_text("word " * 100, size=200, overlap=200)
+
+
+def test_reconciliation_keeps_cross_chunk_relationship(tmp_path: Path) -> None:
+ settings = Settings(
+ _env_file=None,
+ database_url=f"sqlite:///{tmp_path / 'graph.db'}",
+ auto_create_schema=True,
+ artifact_dir=tmp_path / "artifacts",
+ corpus_name="unit",
+ )
+ context = StageContext.create(settings, llm=FakeLLM())
+ context.initialize()
+ with context.storage.session() as session:
+ document = Document(
+ corpus="unit",
+ url="https://example.test/source",
+ text="Acme created Atlas.",
+ content_hash="document",
+ metadata_json={},
+ )
+ session.add(document)
+ session.flush()
+ first = Chunk(
+ document_id=document.id,
+ corpus="unit",
+ ordinal=0,
+ text="Acme created",
+ content_hash="first",
+ )
+ second = Chunk(
+ document_id=document.id,
+ corpus="unit",
+ ordinal=1,
+ text="Atlas",
+ content_hash="second",
+ )
+ session.add_all([first, second])
+ session.flush()
+ session.add_all(
+ [
+ Extraction(
+ chunk_id=first.id,
+ corpus="unit",
+ model_name="test",
+ payload_json={
+ "entities": [
+ {
+ "canonical_name": "Acme",
+ "entity_type": "Organization",
+ "aliases": [],
+ }
+ ],
+ "relationships": [
+ {
+ "source": "Acme",
+ "target": "Atlas",
+ "relationship_type": "CREATED",
+ }
+ ],
+ },
+ ),
+ Extraction(
+ chunk_id=second.id,
+ corpus="unit",
+ model_name="test",
+ payload_json={
+ "entities": [
+ {
+ "canonical_name": "Atlas",
+ "entity_type": "Product",
+ "aliases": [],
+ }
+ ],
+ "relationships": [],
+ },
+ ),
+ ]
+ )
+
+ result = load_stage(5).run(context)
+
+ assert result == {"entities": 2, "relationships": 1}
+ with context.storage.session() as session:
+ relationship = session.scalar(select(ResolvedRelationship))
+ assert relationship is not None
+ assert relationship.source_key == "acme"
+ assert relationship.target_key == "atlas"
diff --git a/tests/unit/test_sources.py b/tests/unit/test_sources.py
new file mode 100644
index 0000000..462b244
--- /dev/null
+++ b/tests/unit/test_sources.py
@@ -0,0 +1,53 @@
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+import pytest
+
+from graphrag_platform.sources import JsonLinesSourceReader
+
+
+def test_jsonl_source_contract_preserves_nested_metadata(tmp_path: Path) -> None:
+ source = tmp_path / "source.jsonl"
+ source.write_text(
+ json.dumps(
+ {
+ "schema_version": "1.0",
+ "external_id": "record-1",
+ "corpus": "test",
+ "url": "https://example.test/article",
+ "body": "Text, with commas and \"quotes\".",
+ "retrieved_at": "2026-10-13T18:00:00Z",
+ "metadata": {"tags": ["one", "two"], "nested": {"value": 1}},
+ }
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+
+ records = list(JsonLinesSourceReader(source).records())
+
+ assert records[0].external_id == "record-1"
+ assert records[0].metadata["tags"] == ["one", "two"]
+ assert '"quotes"' in records[0].body
+
+
+def test_jsonl_source_contract_rejects_unknown_schema(tmp_path: Path) -> None:
+ source = tmp_path / "source.jsonl"
+ source.write_text(
+ json.dumps(
+ {
+ "schema_version": "2.0",
+ "external_id": "record-1",
+ "url": "https://example.test/article",
+ "body": "text",
+ "retrieved_at": "2026-10-13T18:00:00Z",
+ }
+ )
+ + "\n",
+ encoding="utf-8",
+ )
+
+ with pytest.raises(ValueError, match="unsupported source schema"):
+ list(JsonLinesSourceReader(source).records())