moved to this repo

This commit is contained in:
GAJ3T 2026-10-13 19:21:52 -04:00
parent 95d328615d
commit 964b043a94
49 changed files with 4943 additions and 414 deletions

13
.dockerignore Normal file
View File

@ -0,0 +1,13 @@
.env
.git
.github
.venv
__pycache__
*.py[cod]
.pytest_cache
.coverage
artifacts
build
dist
*.egg-info
tests

47
.env.example Normal file
View File

@ -0,0 +1,47 @@
# Storage. Replace the host, user, password, and database for each environment.
POSTGRES_PASSWORD=knowledgegraph-dev
POSTGRES_PORT=5432
API_PORT=8080
DATABASE_URL=postgresql+psycopg://knowledgegraph_app:knowledgegraph-dev@localhost:5432/knowledgegraph
SOURCE_DATABASE_URL=
SOURCE_TABLE_NAME=external_documents
ANALYTICS_DATABASE_URL=
AUTO_CREATE_SCHEMA=false
# External handoff and corpus.
CORPUS_NAME=product-knowledge
# Runtime model templates. JSON files passed with --model-file override these values.
ENTITY_TYPES=Person,Organization,Product,Concept,Event,Location
RELATIONSHIP_TYPES=AFFILIATED_WITH,CREATED,DEPENDS_ON,MENTIONS,PART_OF,RELATED_TO
NODE_TEMPLATE={"canonical_name":"string","entity_type":"one configured entity type","description":"source-grounded string","aliases":["string"]}
EDGE_TEMPLATE={"source":"entity name","target":"entity name","relationship_type":"one configured relationship type","description":"source-grounded string"}
EXTRACTION_INSTRUCTIONS=Extract only entities and relationships explicitly supported by the supplied text.
# OpenAI-compatible chat and embedding provider.
LLM_BASE_URL=https://api.openai.com/v1
LLM_API_KEY=replace-with-provider-key
LLM_CHAT_MODEL=gpt-4.1-mini
LLM_EMBEDDING_MODEL=text-embedding-3-small
LLM_EMBEDDING_DIMENSIONS=1536
LLM_TIMEOUT_SECONDS=60
LLM_MAX_RETRIES=4
LLM_RETRY_INITIAL_SECONDS=1
LLM_RETRY_MAX_SECONDS=30
# Processing and retrieval.
CHUNK_SIZE=1200
CHUNK_OVERLAP=150
COMMUNITY_RESOLUTION=1.0
RETRIEVAL_TOP_K=8
RETRIEVAL_GRAPH_HOPS=2
RETRIEVAL_GRAPH_LIMIT=50
RETRIEVAL_HNSW_EF_SEARCH=100
LOG_LEVEL=INFO
INTERNAL_JWT_SECRET=replace-with-a-dev-test-secret-at-least-32-bytes
INTERNAL_JWT_ISSUER=knowledgegraph-bff
INTERNAL_JWT_AUDIENCE=knowledgegraph-service
INTERNAL_FEATURE_NAME=KnowledgeGraph
WORKER_POLL_SECONDS=2
WORKER_STALE_RUN_SECONDS=3600
ARTIFACT_DIR=artifacts

56
.github/workflows/ci.yml vendored Normal file
View File

@ -0,0 +1,56 @@
name: CI
on:
push:
pull_request:
jobs:
test:
runs-on: ubuntu-latest
services:
postgres:
image: pgvector/pgvector:pg16
env:
POSTGRES_USER: knowledgegraph_app
POSTGRES_PASSWORD: test-password
POSTGRES_DB: knowledgegraph
ports:
- 5432:5432
options: >-
--health-cmd "pg_isready -U knowledgegraph_app -d knowledgegraph"
--health-interval 5s
--health-timeout 5s
--health-retries 20
env:
DATABASE_URL: postgresql+psycopg://knowledgegraph_app:test-password@localhost:5432/knowledgegraph
TEST_POSTGRES_URL: postgresql+psycopg://knowledgegraph_app:test-password@localhost:5432/knowledgegraph
steps:
- uses: actions/checkout@v4
- uses: actions/setup-python@v5
with:
python-version: "3.12"
cache: pip
- name: Install
run: python -m pip install -e ".[test]"
- name: Compile
run: python -m compileall -q src tests migrations
- name: Check dependencies
run: python -m pip check
- name: Run portable tests
run: python -m pytest -q --ignore=tests/integration/test_pgvector_postgres.py
- name: Run migration and PgVector integration
run: python -m pytest tests/integration/test_pgvector_postgres.py -q
- name: Validate Compose
run: docker compose config --quiet
- name: Build application image
run: docker build -f Containerfile -t knowledgegraph:test .

424
.gitignore vendored
View File

@ -1,414 +1,12 @@
# ---> VisualStudio
## Ignore Visual Studio temporary files, build results, and
## files generated by popular Visual Studio add-ons.
##
## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore
# User-specific files
*.rsuser
*.suo
*.user
*.userosscache
*.sln.docstates
# User-specific files (MonoDevelop/Xamarin Studio)
*.userprefs
# Mono auto generated files
mono_crash.*
# Build results
[Dd]ebug/
[Dd]ebugPublic/
[Rr]elease/
[Rr]eleases/
x64/
x86/
[Ww][Ii][Nn]32/
[Aa][Rr][Mm]/
[Aa][Rr][Mm]64/
bld/
[Bb]in/
[Oo]bj/
[Ll]og/
[Ll]ogs/
# Visual Studio 2015/2017 cache/options directory
.vs/
# Uncomment if you have tasks that create the project's static files in wwwroot
#wwwroot/
# Visual Studio 2017 auto generated files
Generated\ Files/
# MSTest test Results
[Tt]est[Rr]esult*/
[Bb]uild[Ll]og.*
# NUnit
*.VisualState.xml
TestResult.xml
nunit-*.xml
# Build Results of an ATL Project
[Dd]ebugPS/
[Rr]eleasePS/
dlldata.c
# Benchmark Results
BenchmarkDotNet.Artifacts/
# .NET Core
project.lock.json
project.fragment.lock.json
artifacts/
# ASP.NET Scaffolding
ScaffoldingReadMe.txt
# StyleCop
StyleCopReport.xml
# Files built by Visual Studio
*_i.c
*_p.c
*_h.h
*.ilk
*.meta
*.obj
*.iobj
*.pch
*.pdb
*.ipdb
*.pgc
*.pgd
*.rsp
*.sbr
*.tlb
*.tli
*.tlh
*.tmp
*.tmp_proj
*_wpftmp.csproj
*.log
*.tlog
*.vspscc
*.vssscc
.builds
*.pidb
*.svclog
*.scc
# Chutzpah Test files
_Chutzpah*
# Visual C++ cache files
ipch/
*.aps
*.ncb
*.opendb
*.opensdf
*.sdf
*.cachefile
*.VC.db
*.VC.VC.opendb
# Visual Studio profiler
*.psess
*.vsp
*.vspx
*.sap
# Visual Studio Trace Files
*.e2e
# TFS 2012 Local Workspace
$tf/
# Guidance Automation Toolkit
*.gpState
# ReSharper is a .NET coding add-in
_ReSharper*/
*.[Rr]e[Ss]harper
*.DotSettings.user
# TeamCity is a build add-in
_TeamCity*
# DotCover is a Code Coverage Tool
*.dotCover
# AxoCover is a Code Coverage Tool
.axoCover/*
!.axoCover/settings.json
# Coverlet is a free, cross platform Code Coverage Tool
coverage*.json
coverage*.xml
coverage*.info
# Visual Studio code coverage results
*.coverage
*.coveragexml
# NCrunch
_NCrunch_*
.*crunch*.local.xml
nCrunchTemp_*
# MightyMoose
*.mm.*
AutoTest.Net/
# Web workbench (sass)
.sass-cache/
# Installshield output folder
[Ee]xpress/
# DocProject is a documentation generator add-in
DocProject/buildhelp/
DocProject/Help/*.HxT
DocProject/Help/*.HxC
DocProject/Help/*.hhc
DocProject/Help/*.hhk
DocProject/Help/*.hhp
DocProject/Help/Html2
DocProject/Help/html
# Click-Once directory
publish/
# Publish Web Output
*.[Pp]ublish.xml
*.azurePubxml
# Note: Comment the next line if you want to checkin your web deploy settings,
# but database connection strings (with potential passwords) will be unencrypted
*.pubxml
*.publishproj
# Microsoft Azure Web App publish settings. Comment the next line if you want to
# checkin your Azure Web App publish settings, but sensitive information contained
# in these scripts will be unencrypted
PublishScripts/
# NuGet Packages
*.nupkg
# NuGet Symbol Packages
*.snupkg
# The packages folder can be ignored because of Package Restore
**/[Pp]ackages/*
# except build/, which is used as an MSBuild target.
!**/[Pp]ackages/build/
# Uncomment if necessary however generally it will be regenerated when needed
#!**/[Pp]ackages/repositories.config
# NuGet v3's project.json files produces more ignorable files
*.nuget.props
*.nuget.targets
# Microsoft Azure Build Output
csx/
*.build.csdef
# Microsoft Azure Emulator
ecf/
rcf/
# Windows Store app package directories and files
AppPackages/
BundleArtifacts/
Package.StoreAssociation.xml
_pkginfo.txt
*.appx
*.appxbundle
*.appxupload
# Visual Studio cache files
# files ending in .cache can be ignored
*.[Cc]ache
# but keep track of directories ending in .cache
!?*.[Cc]ache/
# Others
ClientBin/
~$*
*~
*.dbmdl
*.dbproj.schemaview
*.jfm
*.pfx
*.publishsettings
orleans.codegen.cs
# Including strong name files can present a security risk
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
#*.snk
# Since there are multiple workflows, uncomment next line to ignore bower_components
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
#bower_components/
# RIA/Silverlight projects
Generated_Code/
# Backup & report files from converting an old project file
# to a newer Visual Studio version. Backup files are not needed,
# because we have git ;-)
_UpgradeReport_Files/
Backup*/
UpgradeLog*.XML
UpgradeLog*.htm
ServiceFabricBackup/
*.rptproj.bak
# SQL Server files
*.mdf
*.ldf
*.ndf
# Business Intelligence projects
*.rdl.data
*.bim.layout
*.bim_*.settings
*.rptproj.rsuser
*- [Bb]ackup.rdl
*- [Bb]ackup ([0-9]).rdl
*- [Bb]ackup ([0-9][0-9]).rdl
# Microsoft Fakes
FakesAssemblies/
# GhostDoc plugin setting file
*.GhostDoc.xml
# Node.js Tools for Visual Studio
.ntvs_analysis.dat
node_modules/
# Visual Studio 6 build log
*.plg
# Visual Studio 6 workspace options file
*.opt
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
*.vbw
# Visual Studio 6 auto-generated project file (contains which files were open etc.)
*.vbp
# Visual Studio 6 workspace and project file (working project files containing files to include in project)
*.dsw
*.dsp
# Visual Studio 6 technical files
*.ncb
*.aps
# Visual Studio LightSwitch build output
**/*.HTMLClient/GeneratedArtifacts
**/*.DesktopClient/GeneratedArtifacts
**/*.DesktopClient/ModelManifest.xml
**/*.Server/GeneratedArtifacts
**/*.Server/ModelManifest.xml
_Pvt_Extensions
# Paket dependency manager
.paket/paket.exe
paket-files/
# FAKE - F# Make
.fake/
# CodeRush personal settings
.cr/personal
# Python Tools for Visual Studio (PTVS)
.venv/
__pycache__/
*.pyc
# Cake - Uncomment if you are using it
# tools/**
# !tools/packages.config
# Tabs Studio
*.tss
# Telerik's JustMock configuration file
*.jmconfig
# BizTalk build output
*.btp.cs
*.btm.cs
*.odx.cs
*.xsd.cs
# OpenCover UI analysis results
OpenCover/
# Azure Stream Analytics local run output
ASALocalRun/
# MSBuild Binary and Structured Log
*.binlog
# NVidia Nsight GPU debugger configuration file
*.nvuser
# MFractors (Xamarin productivity tool) working folder
.mfractor/
# Local History for Visual Studio
.localhistory/
# Visual Studio History (VSHistory) files
.vshistory/
# BeatPulse healthcheck temp database
healthchecksdb
# Backup folder for Package Reference Convert tool in Visual Studio 2017
MigrationBackup/
# Ionide (cross platform F# VS Code tools) working folder
.ionide/
# Fody - auto-generated XML schema
FodyWeavers.xsd
# VS Code files for those working on multiple tools
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
*.code-workspace
# Local History for Visual Studio Code
.history/
# Windows Installer files from build outputs
*.cab
*.msi
*.msix
*.msm
*.msp
# JetBrains Rider
*.sln.iml
# ---> VisualStudioCode
.vscode/*
!.vscode/settings.json
!.vscode/tasks.json
!.vscode/launch.json
!.vscode/extensions.json
!.vscode/*.code-snippets
# Local History for Visual Studio Code
.history/
# Built Visual Studio Code Extensions
*.vsix
*.py[cod]
.pytest_cache/
.coverage
htmlcov/
.env
*.db
artifacts/
dist/
build/
*.egg-info/

26
Containerfile Normal file
View File

@ -0,0 +1,26 @@
FROM python:3.12-slim AS runtime
ENV PYTHONDONTWRITEBYTECODE=1 \
PYTHONUNBUFFERED=1 \
PIP_NO_CACHE_DIR=1
RUN groupadd --system knowledgegraph \
&& useradd --system --gid knowledgegraph --create-home knowledgegraph
WORKDIR /app
COPY pyproject.toml README.md alembic.ini ./
COPY migrations ./migrations
COPY src ./src
RUN python -m pip install --upgrade pip \
&& python -m pip install .
RUN mkdir -p /app/artifacts \
&& chown -R knowledgegraph:knowledgegraph /app
USER knowledgegraph
EXPOSE 8080
CMD ["uvicorn", "graphrag_platform.api:app", "--host", "0.0.0.0", "--port", "8080"]

642
README.md
View File

@ -1,2 +1,642 @@
# RAG-POC
# KnowledgeGraph GraphRAG
An internal, restartable GraphRAG pipeline that normalizes externally collected
content, extracts and reconciles knowledge, builds a graph, indexes graph and source
evidence with PostgreSQL/PgVector, retrieves graph-aware context, and generates
source-cited answers.
The collection process is outside this project. Existing **Web-Access MCP** and
**Data-Store MCP** processes discover and persist source material. This project
starts by validating and normalizing their versioned handoff.
## Architecture
```text
Web-Access MCP + Data-Store MCP
|
| versioned JSONL or PostgreSQL handoff
v
01 normalize and parse
v
02 clean, deduplicate, and chunk
v
03 define knowledge model
v
04 extract entities and relationships
v
05 reconcile aliases and duplicates
v
06 materialize graph nodes and edges
v
07 detect and summarize communities
v
08 build PgVector HNSW retrieval index
v
09 retrieve vector + graph + source context
v
10 generate a cited answer
```
PostgreSQL is the canonical checkpoint store. Every completed build stage persists
its output, so later stages can be rebuilt without repeating valid upstream work.
PgVector stores embeddings and performs server-side HNSW cosine search. Indexed
relational edge tables provide bounded graph expansion.
The deployment consists of:
- PostgreSQL 16 with PgVector
- Alembic migration job
- Internal FastAPI service
- PostgreSQL-backed asynchronous worker
- Existing BFF as the policy enforcement point
The API is internal only and is not intended for direct customer access.
## Repository layout
| Path | Purpose |
|---|---|
| [`src/01_normalize_content.py`](src/01_normalize_content.py) through [`src/10_generate_answer.py`](src/10_generate_answer.py) | Complete numbered stage implementations |
| [`src/graphrag_platform`](src/graphrag_platform) | Configuration, models, storage, orchestration, API, auth, worker, and provider clients |
| [`migrations`](migrations) | Alembic database migrations |
| [`tests`](tests) | Unit, API/worker, pipeline, migration, and real PgVector integration tests |
| [`compose.yaml`](compose.yaml) | Reproducible PostgreSQL, migration, API, and worker environment |
| [`Containerfile`](Containerfile) | Non-root application image |
| [`.env.example`](.env.example) | Runtime configuration template |
## Prerequisites
- Python 3.11 or newer
- Podman 5 or newer
- `podman machine` running on Windows
- An OpenAI-compatible chat and embeddings endpoint
- External Web-Access MCP/Data-Store MCP collection process
## Quick start with Compose
Create the local environment:
```powershell
Copy-Item .env.example .env
```
Edit `.env` and set at minimum:
```dotenv
POSTGRES_PASSWORD=choose-a-local-password
DATABASE_URL=postgresql+psycopg://knowledgegraph_app:choose-a-local-password@localhost:5432/knowledgegraph
LLM_BASE_URL=https://api.openai.com/v1
LLM_API_KEY=replace-with-provider-key
LLM_CHAT_MODEL=gpt-4.1-mini
LLM_EMBEDDING_MODEL=text-embedding-3-small
LLM_EMBEDDING_DIMENSIONS=1536
INTERNAL_JWT_SECRET=replace-with-at-least-32-random-bytes
```
`POSTGRES_PASSWORD` and the password in `DATABASE_URL` must match. Compose replaces
the database host with `postgres` inside application containers.
Start the complete environment:
```powershell
podman machine start
podman compose up --detach --build
```
Compose:
1. Starts `docker.io/pgvector/pgvector:pg16`.
2. Waits for PostgreSQL readiness.
3. Runs `alembic upgrade head`.
4. Starts the API and worker.
5. Waits for API dependency readiness.
Verify it:
```powershell
podman compose ps
Invoke-RestMethod http://127.0.0.1:8080/internal/knowledge-graph/health
Invoke-RestMethod http://127.0.0.1:8080/internal/knowledge-graph/ready
```
View logs:
```powershell
podman compose logs --follow api worker
```
Stop services while retaining PostgreSQL data:
```powershell
podman compose down
```
Delete services and all local database data:
```powershell
podman compose down --volumes
```
Volume removal is destructive.
## External source handoff
### JSON Lines contract
Each UTF-8 JSONL line must satisfy schema version `1.0`:
```json
{
"schema_version": "1.0",
"external_id": "stable-source-record-id",
"corpus": "product-knowledge",
"url": "https://docs.example.com/article",
"body": "<html><body><main>Article text</main></body></html>",
"content_type": "text/html",
"status_code": 200,
"retrieved_at": "2026-10-13T18:00:00Z",
"metadata": {
"collector": "Web-Access",
"stored_by": "Data-Store"
}
}
```
Required fields:
- `schema_version`: currently `1.0`
- `external_id`: stable and unique within a corpus
- `url`
- `body`
- `retrieved_at`: ISO 8601 timestamp with offset
Optional/defaulted fields:
- `corpus`: when provided, it must match the selected runtime corpus
- `content_type`: defaults to `text/html`
- `status_code`: defaults to `200`
- `metadata`: defaults to an empty object
Unknown fields and unsupported schema versions are rejected. Invalid records include
the file and line number in the error.
The template is [`src/templates/mcp_documents.jsonl`](src/templates/mcp_documents.jsonl).
A larger acceptance corpus is
[`tests/fixtures/external_documents.jsonl`](tests/fixtures/external_documents.jsonl).
### PostgreSQL source contract
Set `SOURCE_DATABASE_URL` and `SOURCE_TABLE_NAME` to read the same contract from an
external PostgreSQL table:
```dotenv
SOURCE_DATABASE_URL=postgresql+psycopg://source_reader:password@source-host:5432/source_database
SOURCE_TABLE_NAME=external_documents
```
Compatible table:
```sql
CREATE TABLE external_documents (
external_id text PRIMARY KEY,
schema_version text NOT NULL DEFAULT '1.0',
corpus text,
url text NOT NULL,
body text NOT NULL,
content_type text NOT NULL DEFAULT 'text/html',
status_code integer NOT NULL DEFAULT 200,
retrieved_at timestamptz NOT NULL,
metadata jsonb NOT NULL DEFAULT '{}'::jsonb
);
```
Additional source-table columns are ignored. The application account should receive
read-only access to the source table.
Provide either `--input-jsonl` or `SOURCE_DATABASE_URL`, not both.
## Durable checkpoints
Build stages persist these boundaries:
```text
raw_documents + documents
-> chunks
-> knowledge_models
-> extractions
-> resolved_entities + resolved_relationships
-> nodes + edges
-> communities
-> retrieval_items
```
`pipeline_runs` and `stage_runs` separately record:
- queued/running/succeeded/failed state
- selected stage range
- attempt
- configuration fingerprint
- caller and correlation ID
- stage inputs and outputs
- timestamps and error details
The worker requeues runs abandoned beyond `WORKER_STALE_RUN_SECONDS`. Set this value
above the expected maximum duration of one stage.
PostgreSQL advisory locks allow only one active build for a corpus. If another
worker claims a job for that corpus, the job is returned to the queue instead of
executing conflicting writes.
DataFrames are not stage checkpoints. A Pandas or Polars DataFrame may be used by a
future adapter for one bounded transformation, but every completed result must be
persisted before the next stage starts.
## Database migrations
Runtime services verify the migrated schema and do not silently mutate it.
Apply migrations:
```powershell
alembic upgrade head
```
Inspect the current revision:
```powershell
alembic current
alembic history
```
Create a migration after changing SQLAlchemy models:
```powershell
alembic revision --autogenerate -m "describe schema change"
```
Review generated migrations before applying them. Compose runs upgrades through the
one-shot `migrate` service before starting API and worker services.
`AUTO_CREATE_SCHEMA=true` exists only for isolated tests. Do not enable it in shared
dev, test, or production environments.
## Run stages directly
Install a local editable environment:
```powershell
py -3.12 -m venv .venv
.\.venv\Scripts\Activate.ps1
python -m pip install --upgrade pip
python -m pip install -e ".[test]"
```
Run migrations before the first command:
```powershell
alembic upgrade head
```
### 1. Normalize external content
```powershell
python .\src\01_normalize_content.py `
--corpus product-knowledge `
--input-jsonl .\artifacts\mcp-documents.jsonl
```
This validates the external handoff, stores raw provenance, runs Trafilatura for
HTML, and persists normalized document records.
### 2. Clean, deduplicate, and chunk
```powershell
python .\src\02_clean_content.py --corpus product-knowledge
```
### 3. Define the knowledge model
```powershell
python .\src\03_define_knowledge_model.py `
--corpus product-knowledge `
--model-file .\src\templates\knowledge_model.json
```
### 4. Extract entities and relationships
```powershell
python .\src\04_extract_knowledge.py --corpus product-knowledge
```
LLM calls occur outside long database transactions. Each validated chunk result is
committed independently.
### 5. Reconcile knowledge
```powershell
python .\src\05_reconcile_knowledge.py --corpus product-knowledge
```
### 6. Build graph nodes and edges
```powershell
python .\src\06_build_knowledge_graph.py --corpus product-knowledge
```
### 7. Build communities
```powershell
python .\src\07_build_communities.py --corpus product-knowledge
```
### 8. Build the PgVector index
```powershell
python .\src\08_build_retrieval_index.py --corpus product-knowledge
```
This embeds chunks, nodes, edges, and community summaries, validates dimensions,
stores vectors, and creates a dimension-specific HNSW cosine index.
### 9. Retrieve GraphRAG context
```powershell
python .\src\09_query_graphrag.py `
--corpus product-knowledge `
--question "Who created Atlas?" `
--top-k 8
```
The result contains vector `context`, expanded `graph_context`, and traceable
`sources`.
### 10. Generate a cited answer
```powershell
python .\src\10_generate_answer.py `
--corpus product-knowledge `
--question "Who created Atlas?"
```
## Run a durable build
Run stages 1 through 8 under one durable pipeline record:
```powershell
knowledgegraph run `
--corpus product-knowledge `
--from-stage 1 `
--through-stage 8 `
--input-jsonl .\artifacts\mcp-documents.jsonl `
--model-file .\src\templates\knowledge_model.json `
--requested-by local-developer
```
Use `--from-stage` to rebuild only downstream state. The `--reset` option deletes
the selected stage and downstream derived corpus data before execution.
## BFF authorization contract
The existing BFF authenticates callers and evaluates the `KnowledgeGraph` feature.
Independent permission bits are:
```text
Read = 1
Add = 2
Edit = 4
Delete = 8
Execute = 16
```
| Operation | Required mask |
|---|---:|
| Read job/source/retrieval information | `Read` (`1`) |
| Submit a stage-1 source job | `Add | Execute` (`18`) |
| Run downstream build stages | `Execute` (`16`) |
| Query or answer | `Read | Execute` (`17`) |
| Retry a failed job | `Execute` (`16`) |
The BFF forwards a signed internal JWT. Required claims:
```json
{
"iss": "knowledgegraph-bff",
"aud": "knowledgegraph-service",
"sub": "internal-caller-id",
"iat": 1791930000,
"exp": 1791930600,
"jti": "correlation-or-request-id",
"features": {
"KnowledgeGraph": 31
}
}
```
Dev/test uses HS256 and `INTERNAL_JWT_SECRET`. Production deployments can replace
this verifier with the BFF's workload identity, mTLS, or asymmetric/JWKS contract.
Never trust unsigned permission headers.
## Internal API
| Endpoint | Purpose |
|---|---|
| `GET /internal/knowledge-graph/health` | Process liveness |
| `GET /internal/knowledge-graph/ready` | Database and PgVector readiness |
| `POST /internal/knowledge-graph/jobs` | Queue stages 1–8 |
| `GET /internal/knowledge-graph/jobs/{jobId}` | Read durable job/stage status |
| `POST /internal/knowledge-graph/jobs/{jobId}/retry` | Requeue a failed job |
| `POST /internal/knowledge-graph/query` | Synchronous bounded retrieval |
| `POST /internal/knowledge-graph/answer` | Synchronous cited answer |
Example job request:
```json
{
"corpus": "product-knowledge",
"from_stage": 1,
"through_stage": 8,
"input_jsonl": "/app/artifacts/mcp-documents.jsonl",
"model_file": "/app/src/templates/knowledge_model.json",
"reset": false
}
```
Compose bind-mounts local [`artifacts`](artifacts) at `/app/artifacts`, allowing the
worker to read JSONL handoffs placed there. In deployed environments, prefer the
PostgreSQL source contract or a shared managed file volume.
Long-running stages return `202 Accepted` with a job ID. Query and answer requests
remain synchronous and bounded.
## Retry and logging behavior
Model-provider calls retry:
- HTTP 429
- HTTP 500, 502, 503, and 504
- connection failures
- timeouts
Retries honor numeric `Retry-After`, otherwise use bounded exponential backoff with
jitter. Authentication and validation errors are not retried.
Logs are emitted as one JSON object per line and include event names, run IDs,
corpus/stage context where available, attempts, status, and error details. Secrets
are not logged.
## Tests
Portable tests:
```powershell
pytest --ignore=.\tests\integration\test_pgvector_postgres.py
```
Real PostgreSQL/PgVector migration and retrieval test:
```powershell
$env:TEST_POSTGRES_URL = "postgresql+psycopg://knowledgegraph_app:password@localhost:5432/knowledgegraph"
pytest .\tests\integration\test_pgvector_postgres.py
Remove-Item Env:TEST_POSTGRES_URL
```
The suite covers:
- Versioned source validation
- Nested metadata and quoted/comma-containing content
- Configuration parsing
- Chunk validation
- Cross-chunk relationship reconciliation
- Provider retry policy
- Full ten-stage behavior
- Signed BFF authorization and permission failures
- Durable API job and worker execution
- Alembic migration against PostgreSQL
- PgVector extension and HNSW index creation
- Server-side vector retrieval, graph expansion, and citations
Opt-in vector load test:
```powershell
$env:TEST_POSTGRES_URL = "postgresql+psycopg://knowledgegraph_app:password@localhost:5432/knowledgegraph"
$env:RUN_PERFORMANCE_TESTS = "1"
$env:PERF_VECTOR_ITEM_COUNT = "10000"
$env:PERF_MAX_QUERY_SECONDS = "2.0"
pytest .\tests\performance\test_vector_query_load.py
```
Set the item count and latency threshold to the capacity target for the dev/test
environment. Performance checks are excluded by default because machine resources
vary.
CI is defined in [`.github/workflows/ci.yml`](.github/workflows/ci.yml) and compiles,
tests, validates Compose, exercises real PgVector, and builds the application image.
## Persistent format policy
- PostgreSQL is the canonical operational store.
- JSON Lines is the preferred file handoff/checkpoint format.
- CSV is allowed only for explicitly defined flat import/export views.
CSV safely supports commas, quotes, and line breaks when written by a compliant CSV
library. It remains unsuitable as the canonical graph format because aliases,
metadata, source ID arrays, embeddings, and graph properties are nested and typed.
Never construct CSV by joining strings with commas.
## RAG database alternatives
### PostgreSQL with PgVector
Implemented default. Choose it for SQL transactions, colocated provenance and
vectors, self-managed deployment, and tunable HNSW retrieval. Graph traversal uses
indexed relational edges rather than a native property-graph language.
Official reference: [PgVector](https://github.com/pgvector/pgvector).
### Azure Cosmos DB for NoSQL
Choose it for managed/serverless document storage, vector indexing, and first-party
vector plus BM25 hybrid search. Cosmos DB's Gremlin API is separate from the NoSQL
vector feature; use document adjacency or a separate graph store.
References:
- [Cosmos DB vector search](https://learn.microsoft.com/azure/cosmos-db/vector-search)
- [Cosmos DB hybrid search](https://learn.microsoft.com/azure/cosmos-db/gen-ai/hybrid-search)
- [Cosmos DB Gremlin API](https://learn.microsoft.com/azure/cosmos-db/gremlin/overview)
### Neo4j
Choose it when graph topology, paths, and multi-hop traversal are primary retrieval
signals. Neo4j combines native property graphs, Cypher, vector indexes, and
graph-aware hybrid retrieval. It requires a dedicated backend adapter; changing
`DATABASE_URL` is not sufficient. Confirm version and edition requirements for the
selected vector and unified-search features.
References:
- [Neo4j vector indexes](https://neo4j.com/docs/cypher-manual/current/indexes/search-indexes/vector-indexes/)
- [Neo4j hybrid search](https://neo4j.com/developer/genai-ecosystem/hybrid-search/)
Other credible options:
- [Amazon Neptune Analytics](https://docs.aws.amazon.com/neptune-analytics/latest/userguide/what-is-neptune-analytics.html)
for AWS-managed graph/vector analytics
- [Azure AI Search](https://learn.microsoft.com/azure/search/hybrid-search-overview)
as a secondary managed hybrid index without a native graph model
## Operational guidance
- Keep the API and worker on internal networks accessible only to the BFF and
operators.
- Replace all dev secrets before shared deployment.
- Use a separate PostgreSQL role for migrations and a least-privilege runtime role.
- Back up PostgreSQL before destructive resets or migrations.
- Monitor queued/running/failed run counts, stage duration, LLM retries, database
pool usage, vector latency, and retrieval result counts.
- Set `WORKER_STALE_RUN_SECONDS` above the longest expected single stage.
- Rebuild stage 8 after changing embedding model or dimensions.
- Maintain a golden question/citation set to detect retrieval-quality regressions.
## Troubleshooting
### Database migrations are required
```powershell
alembic current
alembic upgrade head
```
### `LLM_API_KEY is required`
Set `LLM_API_KEY` in `.env` and restart API/worker containers.
### Embedding dimension mismatch
Set `LLM_EMBEDDING_DIMENSIONS` to the exact provider output dimension, then reset
and rerun stage 8.
### Source handoff rejected
Validate every JSONL line against schema version `1.0`. Confirm `corpus`,
`external_id`, URL, body, and `retrieved_at`.
### Job remains queued
```powershell
podman compose ps worker
podman compose logs --tail 100 worker
```
### PostgreSQL/PgVector is not ready
```powershell
podman compose ps postgres
podman compose logs --tail 100 postgres migrate
```
### Port conflict
Change `POSTGRES_PORT` or `API_PORT` in `.env`, then recreate the Compose stack.

38
alembic.ini Normal file
View File

@ -0,0 +1,38 @@
[alembic]
script_location = migrations
prepend_sys_path = .
path_separator = os
[loggers]
keys = root,sqlalchemy,alembic
[handlers]
keys = console
[formatters]
keys = generic
[logger_root]
level = WARN
handlers = console
qualname =
[logger_sqlalchemy]
level = WARN
handlers =
qualname = sqlalchemy.engine
[logger_alembic]
level = INFO
handlers =
qualname = alembic
[handler_console]
class = StreamHandler
args = (sys.stderr,)
level = NOTSET
formatter = generic
[formatter_generic]
format = %(levelname)-5.5s [%(name)s] %(message)s
datefmt = %H:%M:%S

99
compose.yaml Normal file
View File

@ -0,0 +1,99 @@
name: knowledgegraph
services:
postgres:
image: docker.io/pgvector/pgvector:pg16
environment:
POSTGRES_USER: knowledgegraph_app
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-knowledgegraph-dev}
POSTGRES_DB: knowledgegraph
ports:
- "127.0.0.1:${POSTGRES_PORT:-5432}:5432"
volumes:
- knowledgegraph-postgres-data:/var/lib/postgresql/data
healthcheck:
test:
[
"CMD-SHELL",
"pg_isready -U knowledgegraph_app -d knowledgegraph",
]
interval: 5s
timeout: 5s
retries: 20
restart: unless-stopped
migrate:
build:
context: .
dockerfile: Containerfile
command: ["alembic", "upgrade", "head"]
environment: &application-environment
DATABASE_URL: postgresql+psycopg://knowledgegraph_app:${POSTGRES_PASSWORD:-knowledgegraph-dev}@postgres:5432/knowledgegraph
SOURCE_DATABASE_URL: ${SOURCE_DATABASE_URL:-}
SOURCE_TABLE_NAME: ${SOURCE_TABLE_NAME:-external_documents}
AUTO_CREATE_SCHEMA: "false"
CORPUS_NAME: ${CORPUS_NAME:-default-corpus}
LLM_BASE_URL: ${LLM_BASE_URL:-https://api.openai.com/v1}
LLM_API_KEY: ${LLM_API_KEY:-}
LLM_CHAT_MODEL: ${LLM_CHAT_MODEL:-gpt-4.1-mini}
LLM_EMBEDDING_MODEL: ${LLM_EMBEDDING_MODEL:-text-embedding-3-small}
LLM_EMBEDDING_DIMENSIONS: ${LLM_EMBEDDING_DIMENSIONS:-1536}
INTERNAL_JWT_SECRET: ${INTERNAL_JWT_SECRET:-knowledgegraph-dev-secret-change-this-32}
INTERNAL_JWT_ISSUER: ${INTERNAL_JWT_ISSUER:-knowledgegraph-bff}
INTERNAL_JWT_AUDIENCE: ${INTERNAL_JWT_AUDIENCE:-knowledgegraph-service}
INTERNAL_FEATURE_NAME: ${INTERNAL_FEATURE_NAME:-KnowledgeGraph}
LOG_LEVEL: ${LOG_LEVEL:-INFO}
depends_on:
postgres:
condition: service_healthy
restart: "no"
api:
build:
context: .
dockerfile: Containerfile
command:
[
"uvicorn",
"graphrag_platform.api:app",
"--host",
"0.0.0.0",
"--port",
"8080",
]
environment: *application-environment
ports:
- "127.0.0.1:${API_PORT:-8080}:8080"
volumes:
- ./artifacts:/app/artifacts
depends_on:
migrate:
condition: service_completed_successfully
healthcheck:
test:
[
"CMD",
"python",
"-c",
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/internal/knowledge-graph/ready', timeout=5)",
]
interval: 10s
timeout: 5s
retries: 12
restart: unless-stopped
worker:
build:
context: .
dockerfile: Containerfile
command: ["knowledgegraph-worker"]
environment: *application-environment
volumes:
- ./artifacts:/app/artifacts
depends_on:
migrate:
condition: service_completed_successfully
restart: unless-stopped
volumes:
knowledgegraph-postgres-data:

50
migrations/env.py Normal file
View File

@ -0,0 +1,50 @@
from __future__ import annotations
from logging.config import fileConfig
from alembic import context
from sqlalchemy import engine_from_config, pool
from graphrag_platform.config import Settings
from graphrag_platform.models import Base
config = context.config
if config.config_file_name:
fileConfig(config.config_file_name)
config.set_main_option("sqlalchemy.url", Settings().database_url)
target_metadata = Base.metadata
def run_migrations_offline() -> None:
context.configure(
url=config.get_main_option("sqlalchemy.url"),
target_metadata=target_metadata,
literal_binds=True,
dialect_opts={"paramstyle": "named"},
compare_type=True,
)
with context.begin_transaction():
context.run_migrations()
def run_migrations_online() -> None:
connectable = engine_from_config(
config.get_section(config.config_ini_section, {}),
prefix="sqlalchemy.",
poolclass=pool.NullPool,
)
with connectable.connect() as connection:
context.configure(
connection=connection,
target_metadata=target_metadata,
compare_type=True,
)
with context.begin_transaction():
context.run_migrations()
if context.is_offline_mode():
run_migrations_offline()
else:
run_migrations_online()

24
migrations/script.py.mako Normal file
View File

@ -0,0 +1,24 @@
"""${message}
Revision ID: ${up_revision}
Revises: ${down_revision | comma,n}
Create Date: ${create_date}
"""
from typing import Sequence, Union
from alembic import op
import sqlalchemy as sa
${imports if imports else ""}
revision: str = ${repr(up_revision)}
down_revision: Union[str, None] = ${repr(down_revision)}
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
def upgrade() -> None:
${upgrades if upgrades else "pass"}
def downgrade() -> None:
${downgrades if downgrades else "pass"}

View File

@ -0,0 +1,25 @@
"""Create the initial KnowledgeGraph schema.
Revision ID: 20261013_0001
Revises:
Create Date: 2026-10-13
"""
from alembic import op
from graphrag_platform.models import Base
revision = "20261013_0001"
down_revision = None
branch_labels = None
depends_on = None
def upgrade() -> None:
connection = op.get_bind()
connection.exec_driver_sql("CREATE EXTENSION IF NOT EXISTS vector")
Base.metadata.create_all(bind=connection)
def downgrade() -> None:
Base.metadata.drop_all(bind=op.get_bind())

60
pyproject.toml Normal file
View File

@ -0,0 +1,60 @@
[build-system]
requires = ["hatchling>=1.25"]
build-backend = "hatchling.build"
[project]
name = "knowledgegraph-graphrag"
version = "0.1.0"
description = "Production-oriented KnowledgeGraph ingestion, indexing, retrieval, and answer pipeline"
readme = "README.md"
requires-python = ">=3.11"
dependencies = [
"alembic>=1.13,<2",
"fastapi>=0.115,<1",
"httpx>=0.27,<1",
"networkx>=3.3,<4",
"pgvector>=0.3,<1",
"PyJWT>=2.9,<3",
"pydantic>=2.8,<3",
"pydantic-settings>=2.4,<3",
"psycopg[binary]>=3.2,<4",
"sqlalchemy>=2.0,<3",
"trafilatura>=1.12,<3",
"uvicorn[standard]>=0.32,<1",
]
[project.optional-dependencies]
test = [
"pytest>=8.3,<9",
"pytest-cov>=5,<7",
]
[project.scripts]
knowledgegraph = "graphrag_platform.cli:main_all"
knowledgegraph-worker = "graphrag_platform.worker:main"
[tool.hatch.build.targets.wheel]
packages = [
"src/graphrag_platform",
"src/01_normalize_content.py",
"src/02_clean_content.py",
"src/03_define_knowledge_model.py",
"src/04_extract_knowledge.py",
"src/05_reconcile_knowledge.py",
"src/06_build_knowledge_graph.py",
"src/07_build_communities.py",
"src/08_build_retrieval_index.py",
"src/09_query_graphrag.py",
"src/10_generate_answer.py",
]
[tool.pytest.ini_options]
addopts = "-ra"
testpaths = ["tests"]
markers = [
"performance: opt-in performance and load checks",
]
[tool.coverage.run]
source = ["graphrag_platform"]
branch = true

128
src/01_normalize_content.py Normal file
View File

@ -0,0 +1,128 @@
from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
import trafilatura
from graphrag_platform.cli import stage_main
from graphrag_platform.models import Document, RawDocument
from graphrag_platform.runtime import StageContext
from graphrag_platform.sources import SourceRecord, source_reader
def content_hash(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def extract_document(record: SourceRecord) -> dict[str, object]:
if "html" not in record.content_type.casefold():
text = record.body.strip()
if not text:
raise ValueError(f"source content is empty for {record.url}")
return {
"title": record.metadata.get("title"),
"author": record.metadata.get("author"),
"date": record.metadata.get("published_at"),
"hostname": None,
"sitename": None,
"text": text,
}
extracted = trafilatura.extract(
record.body,
url=str(record.url),
output_format="json",
with_metadata=True,
include_comments=False,
include_tables=True,
favor_precision=True,
)
if not extracted:
raise ValueError(f"could not extract article content from {record.url}")
payload = json.loads(extracted)
if not str(payload.get("text") or "").strip():
raise ValueError(f"extracted content is empty for {record.url}")
return payload
def run(
context: StageContext,
*,
input_jsonl: Path | None = None,
) -> dict[str, int]:
context.initialize()
reader = source_reader(
input_jsonl=input_jsonl,
source_database_url=context.settings.source_database_url,
source_table_name=context.settings.source_table_name,
)
normalized = 0
with context.storage.session() as session:
for record in reader.records():
if record.corpus and record.corpus != context.settings.corpus_name:
raise ValueError(
f"record {record.external_id} targets corpus {record.corpus}, "
f"not {context.settings.corpus_name}"
)
payload = extract_document(record)
text = str(payload["text"]).strip()
metadata = dict(record.metadata)
metadata.update(
{
"external_id": record.external_id,
"source_schema_version": record.schema_version,
"retrieved_at": record.retrieved_at.isoformat(),
"hostname": payload.get("hostname"),
"sitename": payload.get("sitename"),
}
)
raw = context.storage.upsert(
session,
RawDocument,
{
"corpus": context.settings.corpus_name,
"external_id": record.external_id,
},
{
"source_schema_version": record.schema_version,
"url": str(record.url),
"status_code": record.status_code,
"content_type": record.content_type,
"body": record.body,
"metadata_json": record.metadata,
"retrieved_at": record.retrieved_at,
},
)
context.storage.upsert(
session,
Document,
{"raw_document_id": raw.id},
{
"corpus": context.settings.corpus_name,
"url": str(record.url),
"title": payload.get("title"),
"author": payload.get("author"),
"published_at": payload.get("date"),
"text": text,
"content_hash": content_hash(text),
"metadata_json": metadata,
"is_duplicate": False,
"canonical_document_id": None,
},
)
normalized += 1
if normalized == 0:
raise ValueError("source handoff contained no records")
return {"normalized": normalized}
def run_from_args(
context: StageContext, args: argparse.Namespace
) -> dict[str, int]:
return run(context, input_jsonl=args.input_jsonl)
if __name__ == "__main__":
stage_main(1, run_from_args)

103
src/02_clean_content.py Normal file
View File

@ -0,0 +1,103 @@
from __future__ import annotations
import argparse
import hashlib
import re
from sqlalchemy import delete, select
from graphrag_platform.cli import stage_main
from graphrag_platform.models import Chunk, Document
from graphrag_platform.runtime import StageContext
_SPACE_RE = re.compile(r"[ \t]+")
_PARAGRAPH_RE = re.compile(r"\n{3,}")
def content_hash(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def clean_text(text: str) -> str:
lines = [_SPACE_RE.sub(" ", line).strip() for line in text.splitlines()]
return _PARAGRAPH_RE.sub("\n\n", "\n".join(lines)).strip()
def chunk_text(text: str, *, size: int, overlap: int) -> list[str]:
if overlap >= size:
raise ValueError("CHUNK_OVERLAP must be smaller than CHUNK_SIZE")
chunks: list[str] = []
start = 0
while start < len(text):
end = min(start + size, len(text))
if end < len(text):
break_at = text.rfind(" ", start + size // 2, end)
if break_at > start:
end = break_at
chunk = text[start:end].strip()
if chunk:
chunks.append(chunk)
if end >= len(text):
break
start = end - overlap
return chunks
def run(context: StageContext) -> dict[str, int]:
context.initialize()
chunk_count = 0
duplicate_count = 0
with context.storage.session() as session:
documents = session.scalars(
select(Document)
.where(Document.corpus == context.settings.corpus_name)
.order_by(Document.id)
).all()
if not documents:
raise ValueError("stage 2 requires normalized documents")
session.execute(
delete(Chunk).where(Chunk.corpus == context.settings.corpus_name)
)
hashes: dict[str, int] = {}
for document in documents:
text = clean_text(document.text)
digest = content_hash(text)
document.text = text
document.content_hash = digest
if digest in hashes:
document.is_duplicate = True
document.canonical_document_id = hashes[digest]
duplicate_count += 1
continue
document.is_duplicate = False
document.canonical_document_id = None
hashes[digest] = document.id
for ordinal, text_chunk in enumerate(
chunk_text(
text,
size=context.settings.chunk_size,
overlap=context.settings.chunk_overlap,
)
):
session.add(
Chunk(
document_id=document.id,
corpus=context.settings.corpus_name,
ordinal=ordinal,
text=text_chunk,
content_hash=content_hash(text_chunk),
)
)
chunk_count += 1
return {"chunks": chunk_count, "duplicates": duplicate_count}
def run_from_args(
context: StageContext, args: argparse.Namespace
) -> dict[str, int]:
del args
return run(context)
if __name__ == "__main__":
stage_main(2, run_from_args)

View File

@ -0,0 +1,56 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
from typing import Any
from graphrag_platform.cli import stage_main
from graphrag_platform.models import KnowledgeModel
from graphrag_platform.runtime import StageContext
def validate_model_definition(definition: dict[str, Any]) -> None:
for key in ("entity_types", "relationship_types"):
value = definition.get(key)
if not isinstance(value, list) or not value or not all(
isinstance(item, str) and item.strip() for item in value
):
raise ValueError(f"{key} must be a non-empty array of strings")
for key in ("node_template", "edge_template"):
if not isinstance(definition.get(key), dict):
raise ValueError(f"{key} must be a JSON object")
def run(
context: StageContext, *, model_file: Path | None = None
) -> dict[str, int]:
context.initialize()
definition = context.settings.model_definition()
if model_file is not None:
loaded = json.loads(model_file.read_text(encoding="utf-8"))
if not isinstance(loaded, dict):
raise ValueError("knowledge model file must contain a JSON object")
definition.update(loaded)
validate_model_definition(definition)
with context.storage.session() as session:
context.storage.upsert(
session,
KnowledgeModel,
{"corpus": context.settings.corpus_name},
{"definition_json": definition},
)
return {
"entity_types": len(definition["entity_types"]),
"relationship_types": len(definition["relationship_types"]),
}
def run_from_args(
context: StageContext, args: argparse.Namespace
) -> dict[str, int]:
return run(context, model_file=args.model_file)
if __name__ == "__main__":
stage_main(3, run_from_args)

View File

@ -0,0 +1,97 @@
from __future__ import annotations
import argparse
import json
from typing import Any
from sqlalchemy import select
from graphrag_platform.cli import stage_main
from graphrag_platform.models import Chunk, Extraction, KnowledgeModel
from graphrag_platform.runtime import StageContext
def validate_extraction(
payload: dict[str, Any], definition: dict[str, Any]
) -> None:
entities = payload.get("entities")
relationships = payload.get("relationships")
if not isinstance(entities, list) or not isinstance(relationships, list):
raise ValueError("extraction requires entities and relationships arrays")
entity_types = set(definition["entity_types"])
relationship_types = set(definition["relationship_types"])
for entity in entities:
if not isinstance(entity, dict):
raise ValueError("each entity must be an object")
if not str(entity.get("canonical_name", "")).strip():
raise ValueError("entity canonical_name must not be empty")
if entity.get("entity_type") not in entity_types:
raise ValueError(f"unsupported entity type: {entity.get('entity_type')}")
if not isinstance(entity.get("aliases", []), list):
raise ValueError("entity aliases must be an array")
for relationship in relationships:
if not isinstance(relationship, dict):
raise ValueError("each relationship must be an object")
if not str(relationship.get("source", "")).strip() or not str(
relationship.get("target", "")
).strip():
raise ValueError("relationship source and target must not be empty")
if relationship.get("relationship_type") not in relationship_types:
raise ValueError(
"unsupported relationship type: "
f"{relationship.get('relationship_type')}"
)
def run(context: StageContext) -> dict[str, int]:
context.initialize()
with context.storage.session() as session:
model = session.get(KnowledgeModel, context.settings.corpus_name)
chunks = session.scalars(
select(Chunk).where(Chunk.corpus == context.settings.corpus_name)
).all()
if model is None or not chunks:
raise ValueError("stage 4 requires a knowledge model and cleaned chunks")
definition = dict(model.definition_json)
chunk_inputs = [(chunk.id, chunk.text) for chunk in chunks]
system = (
"You extract a source-grounded knowledge graph. Return one JSON object "
'with arrays named "entities" and "relationships". Never infer unsupported facts.'
)
extracted_count = 0
for chunk_id, chunk_text in chunk_inputs:
prompt = json.dumps(
{
"knowledge_model": definition,
"source_chunk_id": chunk_id,
"text": chunk_text,
},
ensure_ascii=False,
)
payload = context.llm.complete_json(system, prompt)
validate_extraction(payload, definition)
with context.storage.session() as session:
context.storage.upsert(
session,
Extraction,
{"chunk_id": chunk_id},
{
"corpus": context.settings.corpus_name,
"payload_json": payload,
"model_name": context.llm.chat_model,
},
)
extracted_count += 1
return {"extracted_chunks": extracted_count}
def run_from_args(
context: StageContext, args: argparse.Namespace
) -> dict[str, int]:
del args
return run(context)
if __name__ == "__main__":
stage_main(4, run_from_args)

View File

@ -0,0 +1,137 @@
from __future__ import annotations
import argparse
import re
from typing import Any
from sqlalchemy import delete, select
from graphrag_platform.cli import stage_main
from graphrag_platform.models import (
Extraction,
ResolvedEntity,
ResolvedRelationship,
)
from graphrag_platform.runtime import StageContext
_KEY_RE = re.compile(r"[^\w]+", re.UNICODE)
def canonical_key(name: str) -> str:
return _KEY_RE.sub(" ", name.casefold()).strip()
def merge_text(current: str, incoming: str) -> str:
current = current.strip()
incoming = incoming.strip()
if not incoming or incoming.casefold() in current.casefold():
return current
if not current:
return incoming
return f"{current}\n{incoming}"
def run(context: StageContext) -> dict[str, int]:
context.initialize()
entity_count = 0
relationship_count = 0
with context.storage.session() as session:
extractions = session.scalars(
select(Extraction).where(
Extraction.corpus == context.settings.corpus_name
)
).all()
if not extractions:
raise ValueError("stage 5 requires extracted knowledge")
session.execute(
delete(ResolvedRelationship).where(
ResolvedRelationship.corpus == context.settings.corpus_name
)
)
session.execute(
delete(ResolvedEntity).where(
ResolvedEntity.corpus == context.settings.corpus_name
)
)
entity_rows: dict[str, dict[str, Any]] = {}
relation_rows: dict[tuple[str, str, str], dict[str, Any]] = {}
pending_relationships: list[tuple[int, dict[str, Any]]] = []
for extraction in extractions:
payload = extraction.payload_json
for entity in payload["entities"]:
name = entity["canonical_name"].strip()
key = canonical_key(name)
row = entity_rows.get(key)
if row is None:
row = {
"canonical_name": name,
"entity_type": entity["entity_type"],
"description": "",
"aliases": set(),
"chunk_ids": set(),
}
entity_rows[key] = row
elif row["entity_type"] != entity["entity_type"]:
raise ValueError(
f"conflicting entity types for {name}: "
f"{row['entity_type']} and {entity['entity_type']}"
)
row["description"] = merge_text(
row["description"], entity.get("description", "")
)
row["aliases"].update(entity.get("aliases", []))
row["aliases"].add(name)
row["chunk_ids"].add(extraction.chunk_id)
for relationship in payload["relationships"]:
pending_relationships.append((extraction.chunk_id, relationship))
for chunk_id, relationship in pending_relationships:
source_key = canonical_key(relationship["source"])
target_key = canonical_key(relationship["target"])
if source_key not in entity_rows or target_key not in entity_rows:
continue
relation_type = relationship["relationship_type"]
identity = (source_key, target_key, relation_type)
row = relation_rows.setdefault(
identity, {"description": "", "chunk_ids": set()}
)
row["description"] = merge_text(
row["description"], relationship.get("description", "")
)
row["chunk_ids"].add(chunk_id)
for key, row in entity_rows.items():
session.add(
ResolvedEntity(
corpus=context.settings.corpus_name,
canonical_key=key,
canonical_name=row["canonical_name"],
entity_type=row["entity_type"],
description=row["description"],
aliases_json=sorted(row["aliases"], key=str.casefold),
source_chunk_ids_json=sorted(row["chunk_ids"]),
)
)
entity_count += 1
for (source_key, target_key, relation_type), row in relation_rows.items():
session.add(
ResolvedRelationship(
corpus=context.settings.corpus_name,
source_key=source_key,
target_key=target_key,
relationship_type=relation_type,
description=row["description"],
source_chunk_ids_json=sorted(row["chunk_ids"]),
)
)
relationship_count += 1
return {"entities": entity_count, "relationships": relationship_count}
def run_from_args(
context: StageContext, args: argparse.Namespace
) -> dict[str, int]:
del args
return run(context)
if __name__ == "__main__":
stage_main(5, run_from_args)

View File

@ -0,0 +1,80 @@
from __future__ import annotations
import argparse
from sqlalchemy import delete, select
from graphrag_platform.cli import stage_main
from graphrag_platform.models import (
Edge,
Node,
ResolvedEntity,
ResolvedRelationship,
)
from graphrag_platform.runtime import StageContext
def run(context: StageContext) -> dict[str, int]:
context.initialize()
with context.storage.session() as session:
entities = session.scalars(
select(ResolvedEntity).where(
ResolvedEntity.corpus == context.settings.corpus_name
)
).all()
relationships = session.scalars(
select(ResolvedRelationship).where(
ResolvedRelationship.corpus == context.settings.corpus_name
)
).all()
if not entities:
raise ValueError("stage 6 requires reconciled entities")
session.execute(
delete(Edge).where(Edge.corpus == context.settings.corpus_name)
)
session.execute(
delete(Node).where(Node.corpus == context.settings.corpus_name)
)
node_ids: dict[str, int] = {}
for entity in entities:
node = Node(
corpus=entity.corpus,
canonical_key=entity.canonical_key,
canonical_name=entity.canonical_name,
entity_type=entity.entity_type,
description=entity.description,
aliases_json=entity.aliases_json,
source_chunk_ids_json=entity.source_chunk_ids_json,
)
session.add(node)
session.flush()
node_ids[entity.canonical_key] = node.id
edge_count = 0
for relationship in relationships:
source_id = node_ids.get(relationship.source_key)
target_id = node_ids.get(relationship.target_key)
if source_id is None or target_id is None:
continue
session.add(
Edge(
corpus=relationship.corpus,
source_node_id=source_id,
target_node_id=target_id,
relationship_type=relationship.relationship_type,
description=relationship.description,
source_chunk_ids_json=relationship.source_chunk_ids_json,
)
)
edge_count += 1
return {"nodes": len(node_ids), "edges": edge_count}
def run_from_args(
context: StageContext, args: argparse.Namespace
) -> dict[str, int]:
del args
return run(context)
if __name__ == "__main__":
stage_main(6, run_from_args)

View File

@ -0,0 +1,97 @@
from __future__ import annotations
import argparse
import json
import networkx as nx
from sqlalchemy import delete, select
from graphrag_platform.cli import stage_main
from graphrag_platform.models import Community, Edge, Node
from graphrag_platform.runtime import StageContext
def run(context: StageContext) -> dict[str, int]:
context.initialize()
with context.storage.session() as session:
nodes = session.scalars(
select(Node).where(Node.corpus == context.settings.corpus_name)
).all()
edges = session.scalars(
select(Edge).where(Edge.corpus == context.settings.corpus_name)
).all()
if not nodes:
raise ValueError("stage 7 requires graph nodes")
graph = nx.Graph()
graph.add_nodes_from(node.id for node in nodes)
graph.add_edges_from(
(edge.source_node_id, edge.target_node_id) for edge in edges
)
if graph.number_of_edges():
groups = list(
nx.community.greedy_modularity_communities(
graph, resolution=context.settings.community_resolution
)
)
else:
groups = [{node.id} for node in nodes]
by_id = {node.id: node for node in nodes}
community_inputs = [
(
index,
sorted(group),
[
{
"name": by_id[node_id].canonical_name,
"type": by_id[node_id].entity_type,
"description": by_id[node_id].description,
}
for node_id in sorted(group)
],
)
for index, group in enumerate(groups, start=1)
]
summaries: list[tuple[int, list[int], str, str]] = []
for index, node_ids, evidence in community_inputs:
summary = context.llm.complete_json(
(
"Summarize a knowledge-graph community using only the supplied "
'evidence. Return JSON with non-empty "title" and "summary" strings.'
),
json.dumps(evidence, ensure_ascii=False),
)
title = str(summary.get("title", "")).strip()
description = str(summary.get("summary", "")).strip()
if not title or not description:
raise ValueError("community summary requires title and summary")
summaries.append((index, node_ids, title, description))
with context.storage.session() as session:
session.execute(
delete(Community).where(
Community.corpus == context.settings.corpus_name
)
)
for index, node_ids, title, description in summaries:
session.add(
Community(
corpus=context.settings.corpus_name,
community_key=f"community-{index}",
title=title,
summary=description,
node_ids_json=node_ids,
)
)
return {"communities": len(groups)}
def run_from_args(
context: StageContext, args: argparse.Namespace
) -> dict[str, int]:
del args
return run(context)
if __name__ == "__main__":
stage_main(7, run_from_args)

View File

@ -0,0 +1,164 @@
from __future__ import annotations
import argparse
from typing import Any
from sqlalchemy import delete, select
from graphrag_platform.cli import stage_main
from graphrag_platform.models import (
Chunk,
Community,
Document,
Edge,
Node,
SearchItem,
)
from graphrag_platform.runtime import StageContext
def run(context: StageContext) -> dict[str, int]:
context.initialize()
with context.storage.session() as session:
chunks = session.scalars(
select(Chunk).where(Chunk.corpus == context.settings.corpus_name)
).all()
nodes = session.scalars(
select(Node).where(Node.corpus == context.settings.corpus_name)
).all()
edges = session.scalars(
select(Edge).where(Edge.corpus == context.settings.corpus_name)
).all()
communities = session.scalars(
select(Community).where(
Community.corpus == context.settings.corpus_name
)
).all()
if not nodes or not chunks:
raise ValueError("stage 8 requires graph nodes and source chunks")
documents = {
document.id: document
for document in session.scalars(
select(Document).where(
Document.corpus == context.settings.corpus_name
)
).all()
}
node_names = {node.id: node.canonical_name for node in nodes}
node_source_chunks = {
node.id: set(node.source_chunk_ids_json) for node in nodes
}
candidates: list[dict[str, Any]] = []
for chunk in chunks:
document = documents[chunk.document_id]
candidates.append(
{
"item_type": "chunk",
"item_key": str(chunk.id),
"text": chunk.text,
"source_url": document.url,
"metadata": {
"document_id": document.id,
"title": document.title,
"chunk_ordinal": chunk.ordinal,
},
}
)
for node in nodes:
candidates.append(
{
"item_type": "node",
"item_key": str(node.id),
"text": (
f"{node.canonical_name} ({node.entity_type}): "
f"{node.description}"
),
"source_url": None,
"metadata": {"source_chunk_ids": node.source_chunk_ids_json},
}
)
for edge in edges:
candidates.append(
{
"item_type": "edge",
"item_key": str(edge.id),
"text": (
f"{node_names[edge.source_node_id]} "
f"{edge.relationship_type} "
f"{node_names[edge.target_node_id]}: {edge.description}"
),
"source_url": None,
"metadata": {
"source_node_id": edge.source_node_id,
"target_node_id": edge.target_node_id,
"relationship_type": edge.relationship_type,
"source_chunk_ids": edge.source_chunk_ids_json,
},
}
)
for community in communities:
community_chunk_ids: set[int] = set()
for node_id in community.node_ids_json:
community_chunk_ids.update(node_source_chunks.get(node_id, set()))
candidates.append(
{
"item_type": "community",
"item_key": str(community.id),
"text": f"{community.title}: {community.summary}",
"source_url": None,
"metadata": {
"node_ids": community.node_ids_json,
"source_chunk_ids": sorted(community_chunk_ids),
},
}
)
embeddings: list[list[float]] = []
for offset in range(0, len(candidates), 64):
embeddings.extend(
context.llm.embed(
[row["text"] for row in candidates[offset : offset + 64]]
)
)
if len(embeddings) != len(candidates):
raise ValueError("embedding provider returned an unexpected row count")
dimensions = context.settings.llm_embedding_dimensions
invalid_dimensions = [
len(embedding) for embedding in embeddings if len(embedding) != dimensions
]
if invalid_dimensions:
raise ValueError(
"embedding provider returned vectors with unexpected dimensions: "
f"expected {dimensions}, received {invalid_dimensions[0]}"
)
with context.storage.session() as session:
session.execute(
delete(SearchItem).where(
SearchItem.corpus == context.settings.corpus_name
)
)
for candidate, embedding in zip(candidates, embeddings, strict=True):
session.add(
SearchItem(
corpus=context.settings.corpus_name,
item_type=candidate["item_type"],
item_key=candidate["item_key"],
text=candidate["text"],
source_url=candidate["source_url"],
metadata_json=candidate["metadata"],
embedding=embedding,
embedding_model=context.llm.embedding_model,
)
)
context.storage.ensure_vector_index(context.settings.llm_embedding_dimensions)
return {"indexed_items": len(candidates)}
def run_from_args(
context: StageContext, args: argparse.Namespace
) -> dict[str, int]:
del args
return run(context)
if __name__ == "__main__":
stage_main(8, run_from_args)

238
src/09_query_graphrag.py Normal file
View File

@ -0,0 +1,238 @@
from __future__ import annotations
import argparse
import math
from typing import Any
from pgvector.sqlalchemy import Vector
from sqlalchemy import func, or_, select, text
from sqlalchemy.orm import Session
from graphrag_platform.cli import stage_main
from graphrag_platform.models import Chunk, Document, Edge, Node, SearchItem
from graphrag_platform.runtime import StageContext
def cosine_similarity(left: list[float], right: list[float]) -> float:
if len(left) != len(right) or not left:
return 0.0
denominator = math.sqrt(sum(x * x for x in left)) * math.sqrt(
sum(x * x for x in right)
)
if denominator == 0:
return 0.0
return sum(x * y for x, y in zip(left, right, strict=True)) / denominator
def sources_for_chunks(
session: Session, chunk_ids: set[int]
) -> list[dict[str, Any]]:
if not chunk_ids:
return []
rows = session.execute(
select(Chunk, Document)
.join(Document, Chunk.document_id == Document.id)
.where(Chunk.id.in_(chunk_ids))
.order_by(Document.url, Chunk.ordinal)
).all()
return [
{
"citation": f"S{index}",
"url": document.url,
"title": document.title,
"chunk_id": chunk.id,
"text": chunk.text,
}
for index, (chunk, document) in enumerate(rows, start=1)
]
def rank_vector_context(
context: StageContext,
session: Session,
query_embedding: list[float],
limit: int,
) -> list[tuple[float, SearchItem]]:
if context.storage.supports_vector_search:
dimensions = context.settings.llm_embedding_dimensions
session.execute(
text(
"SET LOCAL hnsw.ef_search = "
f"{context.settings.retrieval_hnsw_ef_search}"
)
)
session.execute(text("SET LOCAL hnsw.iterative_scan = strict_order"))
distance = (
SearchItem.embedding.cast(Vector(dimensions))
.cosine_distance(query_embedding)
.label("distance")
)
rows = session.execute(
select(SearchItem, distance)
.where(
SearchItem.corpus == context.settings.corpus_name,
SearchItem.embedding_model == context.llm.embedding_model,
func.vector_dims(SearchItem.embedding) == dimensions,
)
.order_by(distance)
.limit(limit)
).all()
return [(1.0 - float(distance_value), item) for item, distance_value in rows]
search_items = session.scalars(
select(SearchItem).where(
SearchItem.corpus == context.settings.corpus_name,
SearchItem.embedding_model == context.llm.embedding_model,
)
).all()
return sorted(
(
(cosine_similarity(query_embedding, list(item.embedding)), item)
for item in search_items
),
key=lambda pair: pair[0],
reverse=True,
)[:limit]
def expand_graph(
context: StageContext,
session: Session,
ranked: list[tuple[float, SearchItem]],
) -> tuple[list[dict[str, Any]], set[int]]:
seed_node_ids: set[int] = set()
source_chunk_ids: set[int] = set()
for _, item in ranked:
source_chunk_ids.update(item.metadata_json.get("source_chunk_ids", []))
if item.item_type == "node":
seed_node_ids.add(int(item.item_key))
elif item.item_type == "edge":
seed_node_ids.update(
{
int(item.metadata_json["source_node_id"]),
int(item.metadata_json["target_node_id"]),
}
)
elif item.item_type == "community":
seed_node_ids.update(item.metadata_json.get("node_ids", []))
if not seed_node_ids or context.settings.retrieval_graph_hops == 0:
return [], source_chunk_ids
visited_nodes = set(seed_node_ids)
visited_edges: dict[int, Edge] = {}
frontier = set(seed_node_ids)
for _ in range(context.settings.retrieval_graph_hops):
if not frontier or len(visited_edges) >= context.settings.retrieval_graph_limit:
break
remaining = context.settings.retrieval_graph_limit - len(visited_edges)
edges = session.scalars(
select(Edge)
.where(
Edge.corpus == context.settings.corpus_name,
or_(
Edge.source_node_id.in_(frontier),
Edge.target_node_id.in_(frontier),
),
)
.order_by(Edge.weight.desc(), Edge.id)
.limit(remaining)
).all()
next_frontier: set[int] = set()
for edge in edges:
visited_edges[edge.id] = edge
source_chunk_ids.update(edge.source_chunk_ids_json)
next_frontier.update({edge.source_node_id, edge.target_node_id})
next_frontier -= visited_nodes
visited_nodes.update(next_frontier)
frontier = next_frontier
nodes = {
node.id: node
for node in session.scalars(
select(Node).where(
Node.corpus == context.settings.corpus_name,
Node.id.in_(visited_nodes),
)
).all()
}
for node in nodes.values():
source_chunk_ids.update(node.source_chunk_ids_json)
graph_context = [
{
"edge_id": edge.id,
"source_node_id": edge.source_node_id,
"source": nodes[edge.source_node_id].canonical_name,
"relationship": edge.relationship_type,
"target_node_id": edge.target_node_id,
"target": nodes[edge.target_node_id].canonical_name,
"description": edge.description,
"weight": edge.weight,
"source_chunk_ids": edge.source_chunk_ids_json,
}
for edge in visited_edges.values()
if edge.source_node_id in nodes and edge.target_node_id in nodes
]
return graph_context, source_chunk_ids
def run(
context: StageContext,
*,
question: str,
top_k: int | None = None,
) -> dict[str, Any]:
context.initialize()
if not question.strip():
raise ValueError("question must not be empty")
query_embedding = context.llm.embed([question])[0]
expected_dimensions = context.settings.llm_embedding_dimensions
if len(query_embedding) != expected_dimensions:
raise ValueError(
"embedding provider returned a query vector with unexpected dimensions: "
f"expected {expected_dimensions}, received {len(query_embedding)}"
)
with context.storage.session() as session:
limit = top_k or context.settings.retrieval_top_k
ranked = rank_vector_context(
context,
session,
query_embedding,
limit,
)
if not ranked:
raise ValueError("stage 9 requires a retrieval index")
graph_context, chunk_ids = expand_graph(context, session, ranked)
for _, item in ranked:
if item.item_type == "chunk":
chunk_ids.add(int(item.item_key))
chunk_ids.update(item.metadata_json.get("source_chunk_ids", []))
sources = sources_for_chunks(session, chunk_ids)
return {
"question": question,
"context": [
{
"score": round(score, 6),
"type": item.item_type,
"key": item.item_key,
"text": item.text,
"source_url": item.source_url,
"metadata": item.metadata_json,
}
for score, item in ranked
],
"graph_context": graph_context,
"sources": sources,
}
def run_from_args(
context: StageContext, args: argparse.Namespace
) -> dict[str, Any]:
if not args.question:
raise ValueError("--question is required for stage 9")
return run(context, question=args.question, top_k=args.top_k)
if __name__ == "__main__":
stage_main(9, run_from_args)

47
src/10_generate_answer.py Normal file
View File

@ -0,0 +1,47 @@
from __future__ import annotations
import argparse
import json
from typing import Any
from graphrag_platform.cli import stage_main
from graphrag_platform.runtime import StageContext
from graphrag_platform.stage_loader import load_stage
def run(
context: StageContext,
*,
question: str,
top_k: int | None = None,
) -> dict[str, Any]:
retrieval = load_stage(9).run(
context,
question=question,
top_k=top_k,
)
answer = context.llm.complete_text(
(
"Answer only from the supplied GraphRAG context. Cite supporting sources "
"using [S1], [S2], and so on. If the context does not support an answer, "
"state that clearly. Do not invent citations."
),
json.dumps(retrieval, ensure_ascii=False),
)
return {
"question": question,
"answer": answer,
"sources": retrieval["sources"],
}
def run_from_args(
context: StageContext, args: argparse.Namespace
) -> dict[str, Any]:
if not args.question:
raise ValueError("--question is required for stage 10")
return run(context, question=args.question, top_k=args.top_k)
if __name__ == "__main__":
stage_main(10, run_from_args)

View File

@ -0,0 +1,6 @@
"""KnowledgeGraph GraphRAG platform."""
from graphrag_platform.config import Settings
from graphrag_platform.pipeline import KnowledgeGraphPipeline
__all__ = ["KnowledgeGraphPipeline", "Settings"]

View File

@ -0,0 +1,247 @@
from __future__ import annotations
from typing import Annotated, Any, Callable
import jwt
from fastapi import Depends, FastAPI, HTTPException, Request, status
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
from pydantic import BaseModel, Field
from sqlalchemy import select, text
from graphrag_platform.auth import Permission, Principal, decode_internal_token
from graphrag_platform.config import Settings
from graphrag_platform.llm import LLMClient
from graphrag_platform.models import PipelineRun, StageRun
from graphrag_platform.observability import configure_logging
from graphrag_platform.orchestrator import requeue_failed_run, submit_run
from graphrag_platform.runtime import StageContext
from graphrag_platform.stage_loader import load_stage
from graphrag_platform.storage import Storage
bearer = HTTPBearer(auto_error=False)
class JobRequest(BaseModel):
corpus: str = Field(min_length=1, max_length=200)
from_stage: int = Field(default=1, ge=1, le=8)
through_stage: int = Field(default=8, ge=1, le=8)
input_jsonl: str | None = None
model_file: str | None = None
reset: bool = False
class QueryRequest(BaseModel):
corpus: str = Field(min_length=1, max_length=200)
question: str = Field(min_length=1)
top_k: int | None = Field(default=None, ge=1)
def _principal(
request: Request,
credentials: Annotated[
HTTPAuthorizationCredentials | None, Depends(bearer)
],
) -> Principal:
if credentials is None:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Bearer service token is required",
)
try:
return decode_internal_token(credentials.credentials, request.app.state.settings)
except jwt.PyJWTError as error:
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid service token",
) from error
def require_permissions(
required: Permission,
) -> Callable[[Principal], Principal]:
def dependency(principal: Annotated[Principal, Depends(_principal)]) -> Principal:
if not principal.has(required):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail=f"KnowledgeGraph permission mask {int(required)} is required",
)
return principal
return dependency
def create_app(
settings: Settings | None = None,
*,
llm: LLMClient | None = None,
) -> FastAPI:
active_settings = settings or Settings()
configure_logging(active_settings.log_level)
app = FastAPI(
title="KnowledgeGraph Internal API",
version="1.0.0",
docs_url=None,
redoc_url=None,
openapi_url="/internal/knowledge-graph/openapi.json",
)
app.state.settings = active_settings
app.state.storage = Storage(active_settings.database_url)
app.state.llm = llm
def context(corpus: str) -> StageContext:
corpus_settings = active_settings.model_copy(update={"corpus_name": corpus})
return StageContext.create(
corpus_settings,
storage=app.state.storage,
llm=app.state.llm,
)
@app.get("/internal/knowledge-graph/health")
def health() -> dict[str, str]:
return {"status": "healthy"}
@app.get("/internal/knowledge-graph/ready")
def ready() -> dict[str, str]:
try:
with app.state.storage.engine.connect() as connection:
connection.execute(text("SELECT 1"))
if connection.dialect.name == "postgresql":
extension = connection.scalar(
text(
"SELECT extname FROM pg_extension "
"WHERE extname = 'vector'"
)
)
if extension != "vector":
raise RuntimeError("PgVector extension is not installed")
except Exception as error:
raise HTTPException(
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
detail="Service dependencies are not ready",
) from error
return {"status": "ready"}
@app.post(
"/internal/knowledge-graph/jobs",
status_code=status.HTTP_202_ACCEPTED,
)
def create_job(
body: JobRequest,
principal: Annotated[
Principal, Depends(require_permissions(Permission.EXECUTE))
],
) -> dict[str, str]:
if body.from_stage == 1 and not principal.has(Permission.ADD):
raise HTTPException(
status_code=status.HTTP_403_FORBIDDEN,
detail="KnowledgeGraph Add permission is required for stage 1",
)
if body.from_stage > body.through_stage:
raise HTTPException(
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
detail="from_stage must not exceed through_stage",
)
run_context = context(body.corpus)
run_context.initialize()
run_id = submit_run(
run_context,
from_stage=body.from_stage,
through_stage=body.through_stage,
payload={
"input_jsonl": body.input_jsonl,
"model_file": body.model_file,
"reset": body.reset,
},
requested_by=principal.subject,
correlation_id=principal.correlation_id,
permission_mask=principal.permission_mask,
)
return {"job_id": run_id, "status": "queued"}
@app.get("/internal/knowledge-graph/jobs/{job_id}")
def get_job(
job_id: str,
principal: Annotated[
Principal, Depends(require_permissions(Permission.READ))
],
) -> dict[str, Any]:
del principal
with app.state.storage.session() as session:
run = session.get(PipelineRun, job_id)
if run is None:
raise HTTPException(status_code=404, detail="Job not found")
stages = session.scalars(
select(StageRun)
.where(StageRun.pipeline_run_id == job_id)
.order_by(StageRun.stage_number, StageRun.attempt)
).all()
return {
"job_id": run.id,
"corpus": run.corpus,
"status": run.status,
"attempt": run.attempt,
"from_stage": run.from_stage,
"through_stage": run.through_stage,
"error": run.error_message,
"output": run.output_json,
"stages": [
{
"stage": stage_run.stage_number,
"name": stage_run.stage_name,
"status": stage_run.status,
"attempt": stage_run.attempt,
"output": stage_run.output_json,
"error": stage_run.error_message,
}
for stage_run in stages
],
}
@app.post(
"/internal/knowledge-graph/jobs/{job_id}/retry",
status_code=status.HTTP_202_ACCEPTED,
)
def retry_job(
job_id: str,
principal: Annotated[
Principal, Depends(require_permissions(Permission.EXECUTE))
],
) -> dict[str, str]:
del principal
requeue_failed_run(context(active_settings.corpus_name), job_id)
return {"job_id": job_id, "status": "queued"}
@app.post("/internal/knowledge-graph/query")
def query(
body: QueryRequest,
principal: Annotated[
Principal,
Depends(require_permissions(Permission.READ | Permission.EXECUTE)),
],
) -> dict[str, Any]:
del principal
return load_stage(9).run(
context(body.corpus),
question=body.question,
top_k=body.top_k,
)
@app.post("/internal/knowledge-graph/answer")
def answer(
body: QueryRequest,
principal: Annotated[
Principal,
Depends(require_permissions(Permission.READ | Permission.EXECUTE)),
],
) -> dict[str, Any]:
del principal
return load_stage(10).run(
context(body.corpus),
question=body.question,
top_k=body.top_k,
)
return app
app = create_app()

View File

@ -0,0 +1,56 @@
from __future__ import annotations
from dataclasses import dataclass
from enum import IntFlag
from typing import Any
import jwt
from graphrag_platform.config import Settings
class Permission(IntFlag):
READ = 1
ADD = 2
EDIT = 4
DELETE = 8
EXECUTE = 16
@dataclass(frozen=True)
class Principal:
subject: str
permission_mask: int
correlation_id: str
claims: dict[str, Any]
def has(self, required: Permission) -> bool:
return (self.permission_mask & int(required)) == int(required)
def decode_internal_token(token: str, settings: Settings) -> Principal:
claims = jwt.decode(
token,
settings.internal_jwt_secret,
algorithms=["HS256"],
issuer=settings.internal_jwt_issuer,
audience=settings.internal_jwt_audience,
options={"require": ["exp", "iat", "iss", "aud", "sub", "features"]},
)
features = claims.get("features")
if not isinstance(features, dict):
raise jwt.InvalidTokenError("features claim must be an object")
raw_mask = features.get(settings.internal_feature_name)
if not isinstance(raw_mask, int):
raise jwt.InvalidTokenError(
f"missing integer feature claim: {settings.internal_feature_name}"
)
correlation_id = str(claims.get("correlation_id") or claims.get("jti") or "")
if not correlation_id:
raise jwt.InvalidTokenError("correlation_id or jti is required")
return Principal(
subject=str(claims["sub"]),
permission_mask=raw_mask,
correlation_id=correlation_id,
claims=claims,
)

View File

@ -0,0 +1,111 @@
from __future__ import annotations
import argparse
import json
import sys
from uuid import uuid4
from collections.abc import Callable
from pathlib import Path
from typing import Any
from graphrag_platform.config import Settings
from graphrag_platform.observability import configure_logging
from graphrag_platform.orchestrator import execute_run, submit_run
from graphrag_platform.runtime import StageContext
from graphrag_platform.stage_loader import STAGE_MODULES, load_stage
StageRunner = Callable[[StageContext, argparse.Namespace], dict[str, Any]]
def create_parser(fixed_stage: int | None) -> argparse.ArgumentParser:
parser = argparse.ArgumentParser(
description="Run a KnowledgeGraph GraphRAG pipeline stage."
)
if fixed_stage is None:
parser.add_argument("stage", type=int, choices=range(1, 11))
parser.add_argument("--env-file", type=Path, default=Path(".env"))
parser.add_argument("--database-url", help="Override DATABASE_URL.")
parser.add_argument("--corpus", help="Override CORPUS_NAME.")
parser.add_argument("--input-jsonl", type=Path)
parser.add_argument("--model-file", type=Path)
parser.add_argument("--question")
parser.add_argument("--top-k", type=int)
parser.add_argument("--reset", action="store_true")
return parser
def create_run_parser() -> argparse.ArgumentParser:
parser = create_parser(1)
parser.description = "Run a durable KnowledgeGraph build pipeline."
parser.add_argument("--from-stage", type=int, choices=range(1, 9), default=1)
parser.add_argument("--through-stage", type=int, choices=range(1, 9), default=8)
parser.add_argument("--requested-by", default="local-cli")
parser.add_argument("--correlation-id", default=None)
return parser
def create_context(
args: argparse.Namespace, stage: int, *, apply_reset: bool = True
) -> StageContext:
overrides: dict[str, Any] = {}
if args.database_url:
overrides["database_url"] = args.database_url
if args.corpus:
overrides["corpus_name"] = args.corpus
settings = Settings(_env_file=args.env_file, **overrides)
context = StageContext.create(settings)
context.initialize()
if args.reset and apply_reset:
context.storage.clear_from_stage(settings.corpus_name, stage)
return context
def run(stage: int, args: argparse.Namespace) -> dict[str, Any]:
context = create_context(args, stage)
module = load_stage(stage)
return module.run_from_args(context, args)
def stage_main(stage: int, runner: StageRunner) -> None:
args = create_parser(stage).parse_args()
context = create_context(args, stage)
configure_logging(context.settings.log_level)
result = runner(context, args)
print(json.dumps(result, indent=2, ensure_ascii=False))
def main_all() -> None:
if len(sys.argv) > 1 and sys.argv[1] == "run":
args = create_run_parser().parse_args(sys.argv[2:])
context = create_context(args, args.from_stage, apply_reset=False)
configure_logging(context.settings.log_level)
run_id = submit_run(
context,
from_stage=args.from_stage,
through_stage=args.through_stage,
payload={
"input_jsonl": str(args.input_jsonl) if args.input_jsonl else None,
"model_file": str(args.model_file) if args.model_file else None,
"reset": args.reset,
},
requested_by=args.requested_by,
correlation_id=args.correlation_id or str(uuid4()),
permission_mask=31,
)
outputs = execute_run(context, run_id)
print(json.dumps({"run_id": run_id, "outputs": outputs}, indent=2))
return
args = create_parser(None).parse_args()
configure_logging(Settings(_env_file=args.env_file).log_level)
result = run(args.stage, args)
print(
json.dumps(
{
"stage": args.stage,
"name": STAGE_MODULES[args.stage].removeprefix(f"{args.stage:02d}_"),
**result,
},
indent=2,
ensure_ascii=False,
)
)

View File

@ -0,0 +1,126 @@
from __future__ import annotations
import json
from pathlib import Path
from typing import Annotated, Any
from pydantic import Field, field_validator
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
class Settings(BaseSettings):
"""Runtime configuration loaded from environment variables or a .env file."""
model_config = SettingsConfigDict(
env_file=".env",
env_file_encoding="utf-8",
extra="ignore",
case_sensitive=False,
)
database_url: str = (
"postgresql+psycopg://knowledgegraph_app:change-me"
"@localhost:5432/knowledgegraph"
)
source_database_url: str | None = None
source_table_name: str = "external_documents"
analytics_database_url: str | None = None
auto_create_schema: bool = False
corpus_name: str = "default-corpus"
entity_types: Annotated[list[str], NoDecode] = Field(
default_factory=lambda: [
"Person",
"Organization",
"Product",
"Concept",
"Event",
"Location",
]
)
relationship_types: Annotated[list[str], NoDecode] = Field(
default_factory=lambda: [
"AFFILIATED_WITH",
"CREATED",
"DEPENDS_ON",
"MENTIONS",
"PART_OF",
"RELATED_TO",
]
)
node_template: dict[str, Any] = Field(
default_factory=lambda: {
"canonical_name": "string",
"entity_type": "one configured entity type",
"description": "source-grounded string",
"aliases": ["string"],
}
)
edge_template: dict[str, Any] = Field(
default_factory=lambda: {
"source": "entity name",
"target": "entity name",
"relationship_type": "one configured relationship type",
"description": "source-grounded string",
}
)
extraction_instructions: str = (
"Extract only entities and relationships explicitly supported by the supplied text."
)
llm_base_url: str = "https://api.openai.com/v1"
llm_api_key: str | None = None
llm_chat_model: str = "gpt-4.1-mini"
llm_embedding_model: str = "text-embedding-3-small"
llm_embedding_dimensions: int = Field(default=1536, ge=1)
llm_timeout_seconds: float = Field(default=60, gt=0)
llm_max_retries: int = Field(default=4, ge=0, le=10)
llm_retry_initial_seconds: float = Field(default=1, gt=0)
llm_retry_max_seconds: float = Field(default=30, gt=0)
chunk_size: int = Field(default=1200, ge=200)
chunk_overlap: int = Field(default=150, ge=0)
community_resolution: float = Field(default=1.0, gt=0)
retrieval_top_k: int = Field(default=8, ge=1)
retrieval_graph_hops: int = Field(default=2, ge=0, le=5)
retrieval_graph_limit: int = Field(default=50, ge=1)
retrieval_hnsw_ef_search: int = Field(default=100, ge=1)
log_level: str = "INFO"
internal_jwt_secret: str = "replace-in-dev-test"
internal_jwt_issuer: str = "knowledgegraph-bff"
internal_jwt_audience: str = "knowledgegraph-service"
internal_feature_name: str = "KnowledgeGraph"
worker_poll_seconds: float = Field(default=2, gt=0)
worker_stale_run_seconds: int = Field(default=3600, ge=60)
artifact_dir: Path = Path("artifacts")
@field_validator(
"entity_types",
"relationship_types",
mode="before",
)
@classmethod
def parse_comma_separated_list(cls, value: object) -> object:
if isinstance(value, str):
return [item.strip() for item in value.split(",") if item.strip()]
return value
@field_validator("node_template", "edge_template", mode="before")
@classmethod
def parse_json_object(cls, value: object) -> object:
if isinstance(value, str):
parsed = json.loads(value)
if not isinstance(parsed, dict):
raise ValueError("template must be a JSON object")
return parsed
return value
def model_definition(self) -> dict[str, Any]:
return {
"entity_types": self.entity_types,
"relationship_types": self.relationship_types,
"node_template": self.node_template,
"edge_template": self.edge_template,
"extraction_instructions": self.extraction_instructions,
}

View File

@ -0,0 +1,153 @@
from __future__ import annotations
import json
import logging
import random
import time
from collections.abc import Sequence
from typing import Any, Callable, Protocol
import httpx
from graphrag_platform.config import Settings
LOGGER = logging.getLogger(__name__)
_RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
class LLMClient(Protocol):
chat_model: str
embedding_model: str
def complete_json(self, system: str, user: str) -> dict[str, Any]: ...
def complete_text(self, system: str, user: str) -> str: ...
def embed(self, texts: Sequence[str]) -> list[list[float]]: ...
class OpenAICompatibleClient:
def __init__(
self,
settings: Settings,
transport: httpx.BaseTransport | None = None,
sleeper: Callable[[float], None] = time.sleep,
jitter: Callable[[], float] = random.random,
) -> None:
if not settings.llm_api_key:
raise ValueError("LLM_API_KEY is required for LLM-backed stages")
self.chat_model = settings.llm_chat_model
self.embedding_model = settings.llm_embedding_model
self._settings = settings
self._sleeper = sleeper
self._jitter = jitter
self._client = httpx.Client(
base_url=settings.llm_base_url.rstrip("/") + "/",
timeout=settings.llm_timeout_seconds,
headers={
"Authorization": f"Bearer {settings.llm_api_key}",
"Content-Type": "application/json",
},
transport=transport,
)
def complete_json(self, system: str, user: str) -> dict[str, Any]:
response = self._post(
"chat/completions",
json={
"model": self.chat_model,
"response_format": {"type": "json_object"},
"temperature": 0,
"messages": [
{"role": "system", "content": system},
{"role": "user", "content": user},
],
},
)
content = response.json()["choices"][0]["message"]["content"]
parsed = json.loads(content)
if not isinstance(parsed, dict):
raise ValueError("LLM response must be a JSON object")
return parsed
def complete_text(self, system: str, user: str) -> str:
response = self._post(
"chat/completions",
json={
"model": self.chat_model,
"temperature": 0,
"messages": [
{"role": "system", "content": system},
{"role": "user", "content": user},
],
},
)
return str(response.json()["choices"][0]["message"]["content"]).strip()
def embed(self, texts: Sequence[str]) -> list[list[float]]:
if not texts:
return []
response = self._post(
"embeddings",
json={"model": self.embedding_model, "input": list(texts)},
)
rows = sorted(response.json()["data"], key=lambda row: row["index"])
return [[float(value) for value in row["embedding"]] for row in rows]
def _post(self, path: str, *, json: dict[str, Any]) -> httpx.Response:
attempts = self._settings.llm_max_retries + 1
for attempt in range(1, attempts + 1):
try:
response = self._client.post(path, json=json)
if response.status_code not in _RETRYABLE_STATUS_CODES:
response.raise_for_status()
return response
if attempt == attempts:
response.raise_for_status()
delay = self._retry_delay(response, attempt)
LOGGER.warning(
"Retrying model provider request",
extra={
"event": "llm_retry",
"attempt": attempt,
"status_code": response.status_code,
"delay_seconds": delay,
"path": path,
},
)
self._sleeper(delay)
except (httpx.ConnectError, httpx.TimeoutException) as error:
if attempt == attempts:
raise
delay = self._retry_delay(None, attempt)
LOGGER.warning(
"Retrying model provider request after transport error",
extra={
"event": "llm_retry",
"attempt": attempt,
"error_type": type(error).__name__,
"delay_seconds": delay,
"path": path,
},
)
self._sleeper(delay)
raise RuntimeError("model provider retry loop ended unexpectedly")
def _retry_delay(
self, response: httpx.Response | None, attempt: int
) -> float:
if response is not None:
retry_after = response.headers.get("Retry-After")
if retry_after:
try:
return min(
float(retry_after),
self._settings.llm_retry_max_seconds,
)
except ValueError:
pass
exponential = self._settings.llm_retry_initial_seconds * (2 ** (attempt - 1))
return min(
exponential + self._jitter(),
self._settings.llm_retry_max_seconds,
)

View File

@ -0,0 +1,281 @@
from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from uuid import uuid4
from sqlalchemy import (
DateTime,
Float,
ForeignKey,
Index,
Integer,
JSON,
String,
Text,
UniqueConstraint,
)
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
from pgvector.sqlalchemy import Vector
def utc_now() -> datetime:
return datetime.now(timezone.utc)
class Base(DeclarativeBase):
pass
class RawDocument(Base):
__tablename__ = "raw_documents"
__table_args__ = (
UniqueConstraint("corpus", "external_id", name="uq_raw_corpus_external_id"),
)
id: Mapped[int] = mapped_column(primary_key=True)
corpus: Mapped[str] = mapped_column(String(200), index=True)
external_id: Mapped[str] = mapped_column(String(500))
source_schema_version: Mapped[str] = mapped_column(String(20))
url: Mapped[str] = mapped_column(Text)
content_type: Mapped[str | None] = mapped_column(String(200))
status_code: Mapped[int | None] = mapped_column(Integer)
body: Mapped[str] = mapped_column(Text)
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
collected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
class Document(Base):
__tablename__ = "documents"
__table_args__ = (
UniqueConstraint("corpus", "url", name="uq_document_corpus_url"),
UniqueConstraint("raw_document_id", name="uq_document_raw_document"),
)
id: Mapped[int] = mapped_column(primary_key=True)
raw_document_id: Mapped[int | None] = mapped_column(
ForeignKey("raw_documents.id", ondelete="SET NULL")
)
corpus: Mapped[str] = mapped_column(String(200), index=True)
url: Mapped[str] = mapped_column(Text)
title: Mapped[str | None] = mapped_column(Text)
author: Mapped[str | None] = mapped_column(Text)
published_at: Mapped[str | None] = mapped_column(String(100))
text: Mapped[str] = mapped_column(Text)
content_hash: Mapped[str] = mapped_column(String(64), index=True)
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
is_duplicate: Mapped[bool] = mapped_column(default=False)
canonical_document_id: Mapped[int | None] = mapped_column(
ForeignKey("documents.id", ondelete="SET NULL")
)
class Chunk(Base):
__tablename__ = "chunks"
__table_args__ = (UniqueConstraint("document_id", "ordinal", name="uq_chunk_ordinal"),)
id: Mapped[int] = mapped_column(primary_key=True)
document_id: Mapped[int] = mapped_column(
ForeignKey("documents.id", ondelete="CASCADE"), index=True
)
corpus: Mapped[str] = mapped_column(String(200), index=True)
ordinal: Mapped[int] = mapped_column(Integer)
text: Mapped[str] = mapped_column(Text)
content_hash: Mapped[str] = mapped_column(String(64), index=True)
class KnowledgeModel(Base):
__tablename__ = "knowledge_models"
corpus: Mapped[str] = mapped_column(String(200), primary_key=True)
definition_json: Mapped[dict[str, Any]] = mapped_column(JSON)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=utc_now, onupdate=utc_now
)
class Extraction(Base):
__tablename__ = "extractions"
__table_args__ = (UniqueConstraint("chunk_id", name="uq_extraction_chunk"),)
id: Mapped[int] = mapped_column(primary_key=True)
chunk_id: Mapped[int] = mapped_column(
ForeignKey("chunks.id", ondelete="CASCADE"), index=True
)
corpus: Mapped[str] = mapped_column(String(200), index=True)
payload_json: Mapped[dict[str, Any]] = mapped_column(JSON)
model_name: Mapped[str] = mapped_column(String(200))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
class ResolvedEntity(Base):
__tablename__ = "resolved_entities"
__table_args__ = (
UniqueConstraint("corpus", "canonical_key", name="uq_resolved_entity_key"),
)
id: Mapped[int] = mapped_column(primary_key=True)
corpus: Mapped[str] = mapped_column(String(200), index=True)
canonical_key: Mapped[str] = mapped_column(String(500))
canonical_name: Mapped[str] = mapped_column(Text)
entity_type: Mapped[str] = mapped_column(String(100))
description: Mapped[str] = mapped_column(Text, default="")
aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list)
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
class ResolvedRelationship(Base):
__tablename__ = "resolved_relationships"
__table_args__ = (
UniqueConstraint(
"corpus",
"source_key",
"target_key",
"relationship_type",
name="uq_resolved_relationship",
),
)
id: Mapped[int] = mapped_column(primary_key=True)
corpus: Mapped[str] = mapped_column(String(200), index=True)
source_key: Mapped[str] = mapped_column(String(500))
target_key: Mapped[str] = mapped_column(String(500))
relationship_type: Mapped[str] = mapped_column(String(100))
description: Mapped[str] = mapped_column(Text, default="")
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
class Node(Base):
__tablename__ = "nodes"
__table_args__ = (
UniqueConstraint("corpus", "canonical_key", name="uq_node_canonical_key"),
)
id: Mapped[int] = mapped_column(primary_key=True)
corpus: Mapped[str] = mapped_column(String(200), index=True)
canonical_key: Mapped[str] = mapped_column(String(500))
canonical_name: Mapped[str] = mapped_column(Text)
entity_type: Mapped[str] = mapped_column(String(100), index=True)
description: Mapped[str] = mapped_column(Text, default="")
aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list)
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
class Edge(Base):
__tablename__ = "edges"
__table_args__ = (
UniqueConstraint(
"corpus",
"source_node_id",
"target_node_id",
"relationship_type",
name="uq_edge_identity",
),
)
id: Mapped[int] = mapped_column(primary_key=True)
corpus: Mapped[str] = mapped_column(String(200), index=True)
source_node_id: Mapped[int] = mapped_column(
ForeignKey("nodes.id", ondelete="CASCADE"), index=True
)
target_node_id: Mapped[int] = mapped_column(
ForeignKey("nodes.id", ondelete="CASCADE"), index=True
)
relationship_type: Mapped[str] = mapped_column(String(100), index=True)
description: Mapped[str] = mapped_column(Text, default="")
weight: Mapped[float] = mapped_column(Float, default=1.0)
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
class Community(Base):
__tablename__ = "communities"
__table_args__ = (UniqueConstraint("corpus", "community_key", name="uq_community_key"),)
id: Mapped[int] = mapped_column(primary_key=True)
corpus: Mapped[str] = mapped_column(String(200), index=True)
community_key: Mapped[str] = mapped_column(String(100))
title: Mapped[str] = mapped_column(Text)
summary: Mapped[str] = mapped_column(Text)
node_ids_json: Mapped[list[int]] = mapped_column(JSON)
class SearchItem(Base):
__tablename__ = "retrieval_items"
__table_args__ = (
UniqueConstraint("corpus", "item_type", "item_key", name="uq_search_item"),
Index("ix_retrieval_items_corpus_type", "corpus", "item_type"),
Index(
"ix_retrieval_items_corpus_model",
"corpus",
"embedding_model",
),
)
id: Mapped[int] = mapped_column(primary_key=True)
corpus: Mapped[str] = mapped_column(String(200), index=True)
item_type: Mapped[str] = mapped_column(String(50))
item_key: Mapped[str] = mapped_column(String(100))
text: Mapped[str] = mapped_column(Text)
source_url: Mapped[str | None] = mapped_column(Text)
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
embedding: Mapped[list[float]] = mapped_column(
Vector().with_variant(JSON(), "sqlite")
)
embedding_model: Mapped[str] = mapped_column(String(200))
class PipelineRun(Base):
__tablename__ = "pipeline_runs"
__table_args__ = (
Index("ix_pipeline_runs_status_created", "status", "created_at"),
Index("ix_pipeline_runs_corpus_created", "corpus", "created_at"),
)
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=lambda: str(uuid4())
)
corpus: Mapped[str] = mapped_column(String(200), index=True)
status: Mapped[str] = mapped_column(String(30), index=True, default="queued")
from_stage: Mapped[int] = mapped_column(Integer)
through_stage: Mapped[int] = mapped_column(Integer)
payload_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
configuration_hash: Mapped[str] = mapped_column(String(64))
requested_by: Mapped[str] = mapped_column(String(500))
correlation_id: Mapped[str] = mapped_column(String(200), index=True)
feature_name: Mapped[str] = mapped_column(String(200), default="KnowledgeGraph")
permission_mask: Mapped[int] = mapped_column(Integer)
attempt: Mapped[int] = mapped_column(Integer, default=0)
output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
error_message: Mapped[str | None] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class StageRun(Base):
__tablename__ = "stage_runs"
__table_args__ = (
UniqueConstraint(
"pipeline_run_id",
"stage_number",
"attempt",
name="uq_stage_run_attempt",
),
Index("ix_stage_runs_pipeline_stage", "pipeline_run_id", "stage_number"),
)
id: Mapped[int] = mapped_column(primary_key=True)
pipeline_run_id: Mapped[str] = mapped_column(
ForeignKey("pipeline_runs.id", ondelete="CASCADE"), index=True
)
stage_number: Mapped[int] = mapped_column(Integer)
stage_name: Mapped[str] = mapped_column(String(200))
status: Mapped[str] = mapped_column(String(30), default="running")
attempt: Mapped[int] = mapped_column(Integer)
input_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
error_message: Mapped[str | None] = mapped_column(Text)
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))

View File

@ -0,0 +1,33 @@
from __future__ import annotations
import json
import logging
from datetime import datetime, timezone
from typing import Any
class JsonFormatter(logging.Formatter):
_reserved = set(logging.makeLogRecord({}).__dict__)
def format(self, record: logging.LogRecord) -> str:
payload: dict[str, Any] = {
"timestamp": datetime.now(timezone.utc).isoformat(),
"level": record.levelname,
"logger": record.name,
"message": record.getMessage(),
}
for key, value in record.__dict__.items():
if key not in self._reserved and key not in {"message", "asctime"}:
payload[key] = value
if record.exc_info:
payload["exception"] = self.formatException(record.exc_info)
return json.dumps(payload, ensure_ascii=False, default=str)
def configure_logging(level: str = "INFO") -> None:
handler = logging.StreamHandler()
handler.setFormatter(JsonFormatter())
root = logging.getLogger()
root.handlers.clear()
root.addHandler(handler)
root.setLevel(level.upper())

View File

@ -0,0 +1,330 @@
from __future__ import annotations
import hashlib
import json
import logging
from datetime import datetime, timezone
from pathlib import Path
from typing import Any
from sqlalchemy import select, text, update
from graphrag_platform.models import PipelineRun, StageRun
from graphrag_platform.runtime import StageContext
from graphrag_platform.stage_loader import STAGE_MODULES, load_stage
LOGGER = logging.getLogger(__name__)
BUILD_STAGES = range(1, 9)
class RunDeferredError(RuntimeError):
pass
def utc_now() -> datetime:
return datetime.now(timezone.utc)
def configuration_hash(context: StageContext) -> str:
payload = context.settings.model_dump(
mode="json",
exclude={"llm_api_key", "internal_jwt_secret"},
)
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"))
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
def submit_run(
context: StageContext,
*,
from_stage: int,
through_stage: int,
payload: dict[str, Any],
requested_by: str,
correlation_id: str,
permission_mask: int,
) -> str:
if from_stage not in BUILD_STAGES or through_stage not in BUILD_STAGES:
raise ValueError("pipeline jobs support build stages 1 through 8")
if from_stage > through_stage:
raise ValueError("from_stage must not exceed through_stage")
run = PipelineRun(
corpus=context.settings.corpus_name,
status="queued",
from_stage=from_stage,
through_stage=through_stage,
payload_json=payload,
configuration_hash=configuration_hash(context),
requested_by=requested_by,
correlation_id=correlation_id,
feature_name=context.settings.internal_feature_name,
permission_mask=permission_mask,
)
with context.storage.session() as session:
session.add(run)
session.flush()
run_id = run.id
LOGGER.info(
"Pipeline run queued",
extra={
"event": "pipeline_queued",
"run_id": run_id,
"corpus": context.settings.corpus_name,
"from_stage": from_stage,
"through_stage": through_stage,
"correlation_id": correlation_id,
},
)
return run_id
def claim_next_run(context: StageContext) -> str | None:
with context.storage.session() as session:
statement = (
select(PipelineRun)
.where(PipelineRun.status == "queued")
.order_by(PipelineRun.created_at)
.limit(1)
)
if context.storage.engine.dialect.name == "postgresql":
statement = statement.with_for_update(skip_locked=True)
run = session.scalar(statement)
if run is None:
return None
run.status = "running"
run.attempt += 1
run.started_at = run.started_at or utc_now()
run.heartbeat_at = utc_now()
session.flush()
return run.id
def execute_run(context: StageContext, run_id: str) -> dict[str, Any]:
with context.storage.session() as session:
run = session.get(PipelineRun, run_id)
if run is None:
raise ValueError(f"pipeline run does not exist: {run_id}")
if run.status == "queued":
run.status = "running"
run.attempt += 1
run.started_at = run.started_at or utc_now()
if run.status != "running":
raise ValueError(f"pipeline run {run_id} is {run.status}, not running")
from_stage = run.from_stage
through_stage = run.through_stage
payload = dict(run.payload_json)
attempt = run.attempt
execution_context = context.for_corpus(run.corpus)
corpus = run.corpus
lock_connection = None
if context.storage.engine.dialect.name == "postgresql":
lock_connection = context.storage.engine.connect()
acquired = lock_connection.scalar(
text("SELECT pg_try_advisory_lock(hashtext(:key))"),
{"key": f"knowledgegraph:{corpus}"},
)
if not acquired:
lock_connection.close()
with context.storage.session() as session:
run = session.get(PipelineRun, run_id)
if run is not None:
run.status = "queued"
run.error_message = "Deferred because the corpus is already running"
raise RunDeferredError(f"corpus already has an active run: {corpus}")
outputs: dict[str, Any] = {}
try:
if payload.get("reset") and attempt == 1:
execution_context.storage.clear_from_stage(corpus, from_stage)
for stage_number in range(from_stage, through_stage + 1):
previous_output = _successful_stage_output(
context, run_id, stage_number
)
if previous_output is not None:
outputs[str(stage_number)] = previous_output
continue
stage_name = STAGE_MODULES[stage_number]
stage_run_id = _start_stage_run(
context,
run_id=run_id,
stage_number=stage_number,
stage_name=stage_name,
attempt=attempt,
payload=payload,
)
try:
result = _execute_stage(execution_context, stage_number, payload)
except Exception as error:
_finish_stage_run(
context,
stage_run_id,
status="failed",
error_message=str(error),
)
raise
_finish_stage_run(
context,
stage_run_id,
status="succeeded",
output=result,
)
outputs[str(stage_number)] = result
_checkpoint(context, run_id, outputs)
with context.storage.session() as session:
run = session.get(PipelineRun, run_id)
if run is None:
raise RuntimeError(f"pipeline run disappeared: {run_id}")
run.status = "succeeded"
run.output_json = outputs
run.finished_at = utc_now()
run.heartbeat_at = utc_now()
LOGGER.info(
"Pipeline run succeeded",
extra={"event": "pipeline_succeeded", "run_id": run_id},
)
return outputs
except Exception as error:
with context.storage.session() as session:
run = session.get(PipelineRun, run_id)
if run is not None:
run.status = "failed"
run.error_message = str(error)
run.finished_at = utc_now()
run.heartbeat_at = utc_now()
LOGGER.exception(
"Pipeline run failed",
extra={"event": "pipeline_failed", "run_id": run_id},
)
raise
finally:
if lock_connection is not None:
lock_connection.execute(
text("SELECT pg_advisory_unlock(hashtext(:key))"),
{"key": f"knowledgegraph:{corpus}"},
)
lock_connection.close()
def requeue_failed_run(context: StageContext, run_id: str) -> None:
with context.storage.session() as session:
run = session.get(PipelineRun, run_id)
if run is None:
raise ValueError(f"pipeline run does not exist: {run_id}")
if run.status != "failed":
raise ValueError("only failed pipeline runs can be requeued")
run.status = "queued"
run.error_message = None
run.finished_at = None
def recover_stale_runs(context: StageContext) -> int:
threshold = utc_now().timestamp() - context.settings.worker_stale_run_seconds
cutoff = datetime.fromtimestamp(threshold, tz=timezone.utc)
with context.storage.session() as session:
result = session.execute(
update(PipelineRun)
.where(
PipelineRun.status == "running",
PipelineRun.heartbeat_at < cutoff,
)
.values(
status="queued",
error_message="Recovered after a stale worker lease",
finished_at=None,
)
)
recovered = int(result.rowcount or 0)
if recovered:
LOGGER.warning(
"Recovered stale pipeline runs",
extra={"event": "pipeline_stale_recovered", "count": recovered},
)
return recovered
def _execute_stage(
context: StageContext, stage_number: int, payload: dict[str, Any]
) -> dict[str, Any]:
module = load_stage(stage_number)
if stage_number == 1:
input_path = payload.get("input_jsonl")
return module.run(
context,
input_jsonl=Path(input_path) if input_path else None,
)
if stage_number == 3:
model_path = payload.get("model_file")
return module.run(
context,
model_file=Path(model_path) if model_path else None,
)
return module.run(context)
def _start_stage_run(
context: StageContext,
*,
run_id: str,
stage_number: int,
stage_name: str,
attempt: int,
payload: dict[str, Any],
) -> int:
with context.storage.session() as session:
stage_run = StageRun(
pipeline_run_id=run_id,
stage_number=stage_number,
stage_name=stage_name,
status="running",
attempt=attempt,
input_json=payload,
)
session.add(stage_run)
session.flush()
return stage_run.id
def _finish_stage_run(
context: StageContext,
stage_run_id: int,
*,
status: str,
output: dict[str, Any] | None = None,
error_message: str | None = None,
) -> None:
with context.storage.session() as session:
stage_run = session.get(StageRun, stage_run_id)
if stage_run is None:
raise RuntimeError(f"stage run disappeared: {stage_run_id}")
stage_run.status = status
stage_run.output_json = output or {}
stage_run.error_message = error_message
stage_run.finished_at = utc_now()
def _checkpoint(
context: StageContext, run_id: str, outputs: dict[str, Any]
) -> None:
with context.storage.session() as session:
run = session.get(PipelineRun, run_id)
if run is not None:
run.output_json = outputs
run.heartbeat_at = utc_now()
def _successful_stage_output(
context: StageContext, run_id: str, stage_number: int
) -> dict[str, Any] | None:
with context.storage.session() as session:
stage_run = session.scalar(
select(StageRun)
.where(
StageRun.pipeline_run_id == run_id,
StageRun.stage_number == stage_number,
StageRun.status == "succeeded",
)
.order_by(StageRun.attempt.desc())
.limit(1)
)
return dict(stage_run.output_json) if stage_run is not None else None

View File

@ -0,0 +1,72 @@
from __future__ import annotations
from pathlib import Path
from typing import Any
from graphrag_platform.config import Settings
from graphrag_platform.llm import LLMClient
from graphrag_platform.runtime import StageContext
from graphrag_platform.stage_loader import load_stage
from graphrag_platform.storage import Storage
class KnowledgeGraphPipeline:
"""Compatibility facade over independently executable numbered stages."""
def __init__(
self,
settings: Settings,
*,
storage: Storage | None = None,
llm: LLMClient | None = None,
) -> None:
self.context = StageContext.create(settings, storage=storage, llm=llm)
@property
def settings(self) -> Settings:
return self.context.settings
@property
def storage(self) -> Storage:
return self.context.storage
def initialize(self) -> None:
self.context.initialize()
def stage_01_normalize(
self, *, input_jsonl: Path | None = None
) -> dict[str, int]:
return load_stage(1).run(self.context, input_jsonl=input_jsonl)
def stage_02_clean_and_chunk(self) -> dict[str, int]:
return load_stage(2).run(self.context)
def stage_03_define_model(
self, model_file: Path | None = None
) -> dict[str, int]:
return load_stage(3).run(self.context, model_file=model_file)
def stage_04_extract(self) -> dict[str, int]:
return load_stage(4).run(self.context)
def stage_05_reconcile(self) -> dict[str, int]:
return load_stage(5).run(self.context)
def stage_06_build_graph(self) -> dict[str, int]:
return load_stage(6).run(self.context)
def stage_07_build_communities(self) -> dict[str, int]:
return load_stage(7).run(self.context)
def stage_08_build_retrieval_index(self) -> dict[str, int]:
return load_stage(8).run(self.context)
def stage_09_retrieve(
self, question: str, *, top_k: int | None = None
) -> dict[str, Any]:
return load_stage(9).run(self.context, question=question, top_k=top_k)
def stage_10_answer(
self, question: str, *, top_k: int | None = None
) -> dict[str, Any]:
return load_stage(10).run(self.context, question=question, top_k=top_k)

View File

@ -0,0 +1,50 @@
from __future__ import annotations
from dataclasses import dataclass
from graphrag_platform.config import Settings
from graphrag_platform.llm import LLMClient, OpenAICompatibleClient
from graphrag_platform.storage import Storage
@dataclass
class StageContext:
settings: Settings
storage: Storage
_llm: LLMClient | None = None
@classmethod
def create(
cls,
settings: Settings,
*,
storage: Storage | None = None,
llm: LLMClient | None = None,
) -> StageContext:
return cls(
settings=settings,
storage=storage or Storage(settings.database_url),
_llm=llm,
)
@property
def llm(self) -> LLMClient:
if self._llm is None:
self._llm = OpenAICompatibleClient(self.settings)
return self._llm
def initialize(self) -> None:
if self.settings.auto_create_schema:
self.storage.create_schema()
else:
self.storage.verify_schema()
(self.settings.artifact_dir / self.settings.corpus_name).mkdir(
parents=True, exist_ok=True
)
def for_corpus(self, corpus: str) -> StageContext:
return StageContext.create(
self.settings.model_copy(update={"corpus_name": corpus}),
storage=self.storage,
llm=self._llm,
)

View File

@ -0,0 +1,106 @@
from __future__ import annotations
import json
import re
from collections.abc import Iterator
from datetime import datetime
from pathlib import Path
from typing import Any, Protocol
from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator
from sqlalchemy import MetaData, Table, create_engine, select
_TABLE_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
class SourceRecord(BaseModel):
model_config = ConfigDict(extra="forbid")
schema_version: str = "1.0"
external_id: str = Field(min_length=1, max_length=500)
corpus: str | None = Field(default=None, min_length=1, max_length=200)
url: HttpUrl
body: str = Field(min_length=1)
content_type: str = "text/html"
status_code: int = Field(default=200, ge=100, le=599)
retrieved_at: datetime
metadata: dict[str, Any] = Field(default_factory=dict)
@field_validator("schema_version")
@classmethod
def require_supported_schema(cls, value: str) -> str:
if value != "1.0":
raise ValueError(f"unsupported source schema version: {value}")
return value
class SourceReader(Protocol):
def records(self) -> Iterator[SourceRecord]: ...
class JsonLinesSourceReader:
def __init__(self, path: Path) -> None:
self.path = path
def records(self) -> Iterator[SourceRecord]:
with self.path.open("r", encoding="utf-8") as stream:
for line_number, line in enumerate(stream, start=1):
if not line.strip():
continue
try:
payload = json.loads(line)
yield SourceRecord.model_validate(payload)
except (json.JSONDecodeError, ValueError) as error:
raise ValueError(
f"{self.path}:{line_number}: invalid source record: {error}"
) from error
class PostgreSqlSourceReader:
def __init__(self, database_url: str, table_name: str) -> None:
if not database_url.startswith("postgresql"):
raise ValueError("SOURCE_DATABASE_URL must use PostgreSQL")
if not _TABLE_NAME_RE.fullmatch(table_name):
raise ValueError("SOURCE_TABLE_NAME must be an unqualified SQL identifier")
self.engine = create_engine(database_url, pool_pre_ping=True)
self.table_name = table_name
def records(self) -> Iterator[SourceRecord]:
metadata = MetaData()
table = Table(self.table_name, metadata, autoload_with=self.engine)
required = {
"schema_version",
"external_id",
"url",
"body",
"retrieved_at",
}
missing = required - set(table.columns.keys())
if missing:
raise ValueError(
f"source table {self.table_name} is missing columns: "
f"{', '.join(sorted(missing))}"
)
with self.engine.connect() as connection:
for row in connection.execute(select(table).order_by(table.c.external_id)):
payload = {
key: value
for key, value in row._mapping.items()
if key in SourceRecord.model_fields
}
yield SourceRecord.model_validate(payload)
def source_reader(
*,
input_jsonl: Path | None,
source_database_url: str | None,
source_table_name: str,
) -> SourceReader:
if input_jsonl is not None and source_database_url:
raise ValueError("provide either --input-jsonl or SOURCE_DATABASE_URL, not both")
if input_jsonl is not None:
return JsonLinesSourceReader(input_jsonl)
if source_database_url:
return PostgreSqlSourceReader(source_database_url, source_table_name)
raise ValueError("provide --input-jsonl or configure SOURCE_DATABASE_URL")

View File

@ -0,0 +1,25 @@
from __future__ import annotations
from importlib import import_module
from types import ModuleType
STAGE_MODULES = {
1: "01_normalize_content",
2: "02_clean_content",
3: "03_define_knowledge_model",
4: "04_extract_knowledge",
5: "05_reconcile_knowledge",
6: "06_build_knowledge_graph",
7: "07_build_communities",
8: "08_build_retrieval_index",
9: "09_query_graphrag",
10: "10_generate_answer",
}
def load_stage(stage: int) -> ModuleType:
try:
module_name = STAGE_MODULES[stage]
except KeyError as error:
raise ValueError(f"unsupported stage {stage}") from error
return import_module(module_name)

View File

@ -0,0 +1,140 @@
from __future__ import annotations
from collections.abc import Iterator
from contextlib import contextmanager
from typing import Any, TypeVar
from sqlalchemy import create_engine, delete, event, inspect, select, text
from sqlalchemy.engine import Engine
from sqlalchemy.orm import Session
from graphrag_platform.models import (
Base,
Chunk,
Community,
Document,
Edge,
Extraction,
KnowledgeModel,
Node,
RawDocument,
ResolvedEntity,
ResolvedRelationship,
SearchItem,
)
ModelT = TypeVar("ModelT", bound=Base)
class Storage:
def __init__(self, database_url: str) -> None:
connect_args = {"check_same_thread": False} if database_url.startswith("sqlite") else {}
self.engine: Engine = create_engine(
database_url,
future=True,
pool_pre_ping=True,
connect_args=connect_args,
)
if database_url.startswith("sqlite"):
event.listen(self.engine, "connect", self._enable_sqlite_foreign_keys)
@staticmethod
def _enable_sqlite_foreign_keys(
dbapi_connection: Any, connection_record: Any
) -> None:
del connection_record
cursor = dbapi_connection.cursor()
cursor.execute("PRAGMA foreign_keys=ON")
cursor.close()
def create_schema(self) -> None:
if self.engine.dialect.name == "postgresql":
with self.engine.begin() as connection:
connection.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
Base.metadata.create_all(self.engine)
def verify_schema(self) -> None:
tables = set(inspect(self.engine).get_table_names())
required = {"pipeline_runs", "stage_runs", "documents", "retrieval_items"}
missing = required - tables
if missing:
raise RuntimeError(
"database migrations are required; missing tables: "
f"{', '.join(sorted(missing))}"
)
if self.engine.dialect.name == "postgresql":
with self.engine.connect() as connection:
extension = connection.scalar(
text(
"SELECT extname FROM pg_extension "
"WHERE extname = 'vector'"
)
)
if extension != "vector":
raise RuntimeError("database migrations are required: vector extension")
@property
def supports_vector_search(self) -> bool:
return self.engine.dialect.name == "postgresql"
def ensure_vector_index(self, dimensions: int) -> None:
if not self.supports_vector_search:
return
index_name = f"ix_retrieval_items_embedding_hnsw_{dimensions}"
statement = f"""
CREATE INDEX IF NOT EXISTS {index_name}
ON retrieval_items
USING hnsw ((embedding::vector({dimensions})) vector_cosine_ops)
WHERE vector_dims(embedding) = {dimensions}
"""
with self.engine.begin() as connection:
connection.execute(text(statement))
@contextmanager
def session(self) -> Iterator[Session]:
with Session(self.engine) as session:
try:
yield session
session.commit()
except Exception:
session.rollback()
raise
def upsert(
self,
session: Session,
model: type[ModelT],
identity: dict[str, Any],
values: dict[str, Any],
) -> ModelT:
instance = session.scalar(select(model).filter_by(**identity))
if instance is None:
instance = model(**identity, **values)
session.add(instance)
else:
for key, value in values.items():
setattr(instance, key, value)
session.flush()
return instance
def clear_from_stage(self, corpus: str, stage: int) -> None:
stage_tables: list[tuple[int, type[Base]]] = [
(8, SearchItem),
(7, Community),
(6, Edge),
(6, Node),
(5, ResolvedRelationship),
(5, ResolvedEntity),
(4, Extraction),
(2, Chunk),
(1, RawDocument),
(1, Document),
]
with self.session() as session:
for table_stage, model in stage_tables:
if table_stage >= stage:
session.execute(delete(model).where(model.corpus == corpus))
if stage <= 3:
session.execute(
delete(KnowledgeModel).where(KnowledgeModel.corpus == corpus)
)

View File

@ -0,0 +1,61 @@
from __future__ import annotations
import argparse
import logging
import time
from graphrag_platform.config import Settings
from graphrag_platform.observability import configure_logging
from graphrag_platform.orchestrator import (
RunDeferredError,
claim_next_run,
execute_run,
recover_stale_runs,
)
from graphrag_platform.runtime import StageContext
LOGGER = logging.getLogger(__name__)
def work_once(context: StageContext) -> bool:
run_id = claim_next_run(context)
if run_id is None:
return False
try:
execute_run(context, run_id)
except RunDeferredError:
LOGGER.info(
"Worker deferred a run for an active corpus",
extra={"event": "worker_job_deferred", "run_id": run_id},
)
except Exception:
LOGGER.exception(
"Worker execution failed",
extra={"event": "worker_job_failed", "run_id": run_id},
)
return True
def run_worker(context: StageContext, *, once: bool = False) -> None:
context.initialize()
recover_stale_runs(context)
while True:
worked = work_once(context)
if once:
return
if not worked:
time.sleep(context.settings.worker_poll_seconds)
def main() -> None:
parser = argparse.ArgumentParser(description="Run the KnowledgeGraph job worker.")
parser.add_argument("--env-file", default=".env")
parser.add_argument("--once", action="store_true")
args = parser.parse_args()
settings = Settings(_env_file=args.env_file)
configure_logging(settings.log_level)
run_worker(StageContext.create(settings), once=args.once)
if __name__ == "__main__":
main()

View File

@ -0,0 +1,33 @@
{
"entity_types": [
"Person",
"Organization",
"Product",
"Concept",
"Event",
"Location"
],
"relationship_types": [
"AFFILIATED_WITH",
"CREATED",
"DEPENDS_ON",
"MENTIONS",
"PART_OF",
"RELATED_TO"
],
"node_template": {
"canonical_name": "string",
"entity_type": "one configured entity type",
"description": "source-grounded string",
"aliases": [
"string"
]
},
"edge_template": {
"source": "entity name",
"target": "entity name",
"relationship_type": "one configured relationship type",
"description": "source-grounded string"
},
"extraction_instructions": "Extract only entities and relationships explicitly supported by the supplied text."
}

View File

@ -0,0 +1 @@
{"schema_version":"1.0","external_id":"docs-example-article","corpus":"product-knowledge","url":"https://docs.example.com/article","content_type":"text/html","status_code":200,"retrieved_at":"2026-01-01T00:00:00Z","body":"<html><head><title>Article title</title></head><body><main><h1>Article title</h1><p>Article text collected with Web-Access.</p></main></body></html>","metadata":{"collector":"Web-Access","stored_by":"Data-Store"}}

58
tests/conftest.py Normal file
View File

@ -0,0 +1,58 @@
from __future__ import annotations
import hashlib
from collections.abc import Sequence
from typing import Any
class FakeLLM:
chat_model = "fake-chat-model"
embedding_model = "fake-embedding-model"
def complete_json(self, system: str, user: str) -> dict[str, Any]:
if "community" in system.casefold():
return {
"title": "Atlas product community",
"summary": "Acme created Atlas, a graph retrieval product.",
}
return {
"entities": [
{
"canonical_name": "Acme",
"entity_type": "Organization",
"description": "Acme is a technology organization.",
"aliases": ["Acme Corp"],
},
{
"canonical_name": "Atlas",
"entity_type": "Product",
"description": "Atlas is a graph retrieval product.",
"aliases": [],
},
],
"relationships": [
{
"source": "Acme",
"target": "Atlas",
"relationship_type": "CREATED",
"description": "Acme created Atlas.",
}
],
}
def complete_text(self, system: str, user: str) -> str:
return "Acme created Atlas. [S1]"
def embed(self, texts: Sequence[str]) -> list[list[float]]:
vectors: list[list[float]] = []
for text in texts:
normalized = text.casefold()
digest = hashlib.sha256(normalized.encode("utf-8")).digest()
vectors.append(
[
float("atlas" in normalized),
float("acme" in normalized),
digest[0] / 255,
]
)
return vectors

View File

@ -0,0 +1,3 @@
{"schema_version":"1.0","external_id":"atlas-overview","corpus":"acceptance-corpus","url":"https://docs.example/atlas","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:00:00Z","body":"<html><head><title>Atlas Overview</title></head><body><main><h1>Atlas</h1><p>Acme Corporation created Atlas. Atlas provides graph retrieval for knowledge applications.</p></main></body></html>","metadata":{"collector":"Web-Access","stored_by":"Data-Store","tags":["product","graph"]}}
{"schema_version":"1.0","external_id":"atlas-architecture","corpus":"acceptance-corpus","url":"https://docs.example/atlas/architecture","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:01:00Z","body":"<html><head><title>Atlas Architecture</title></head><body><main><h1>Architecture</h1><p>Acme Corp designed Atlas to combine a knowledge graph with vector retrieval. Atlas depends on PostgreSQL and PgVector.</p></main></body></html>","metadata":{"collector":"Web-Access","stored_by":"Data-Store","description":"Contains commas, \"quotes\", and aliases."}}
{"schema_version":"1.0","external_id":"atlas-overview-copy","corpus":"acceptance-corpus","url":"https://mirror.example/atlas","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:02:00Z","body":"<html><head><title>Atlas Overview</title></head><body><main><h1>Atlas</h1><p>Acme Corporation created Atlas. Atlas provides graph retrieval for knowledge applications.</p></main></body></html>","metadata":{"collector":"Web-Access","stored_by":"Data-Store","duplicate_candidate":true}}

View File

@ -0,0 +1,284 @@
from __future__ import annotations
import json
from datetime import datetime, timedelta, timezone
from pathlib import Path
from uuid import uuid4
import jwt
from fastapi.testclient import TestClient
from graphrag_platform.api import create_app
from graphrag_platform.auth import Permission
from graphrag_platform.config import Settings
from graphrag_platform.runtime import StageContext
from graphrag_platform.worker import work_once
from conftest import FakeLLM
def _token(settings: Settings, mask: int) -> str:
now = datetime.now(timezone.utc)
return jwt.encode(
{
"iss": settings.internal_jwt_issuer,
"aud": settings.internal_jwt_audience,
"sub": "dev-test-client",
"iat": now,
"exp": now + timedelta(minutes=10),
"jti": str(uuid4()),
"features": {settings.internal_feature_name: mask},
},
settings.internal_jwt_secret,
algorithm="HS256",
)
def test_bff_contract_and_worker_execute_durable_job(tmp_path: Path) -> None:
source = tmp_path / "source.jsonl"
source.write_text(
json.dumps(
{
"schema_version": "1.0",
"external_id": "atlas",
"corpus": "api-corpus",
"url": "https://docs.example/atlas",
"retrieved_at": "2026-10-13T18:00:00Z",
"body": (
"<html><head><title>Atlas</title></head><body><main>"
"Acme created Atlas for graph retrieval.</main></body></html>"
),
}
)
+ "\n",
encoding="utf-8",
)
settings = Settings(
_env_file=None,
database_url=f"sqlite:///{tmp_path / 'api.db'}",
auto_create_schema=True,
artifact_dir=tmp_path / "artifacts",
llm_embedding_dimensions=3,
internal_jwt_secret="dev-test-secret-at-least-32-bytes-long",
)
app = create_app(settings, llm=FakeLLM())
full_token = _token(
settings,
int(Permission.READ | Permission.ADD | Permission.EXECUTE),
)
with TestClient(app) as client:
assert client.get("/internal/knowledge-graph/health").status_code == 200
assert (
client.post(
"/internal/knowledge-graph/jobs",
json={
"corpus": "api-corpus",
"from_stage": 1,
"through_stage": 8,
"input_jsonl": str(source),
},
).status_code
== 401
)
execute_only = _token(settings, int(Permission.EXECUTE))
assert (
client.post(
"/internal/knowledge-graph/jobs",
headers={"Authorization": f"Bearer {execute_only}"},
json={
"corpus": "api-corpus",
"from_stage": 1,
"through_stage": 8,
"input_jsonl": str(source),
},
).status_code
== 403
)
queued = client.post(
"/internal/knowledge-graph/jobs",
headers={"Authorization": f"Bearer {full_token}"},
json={
"corpus": "api-corpus",
"from_stage": 1,
"through_stage": 8,
"input_jsonl": str(source),
},
)
assert queued.status_code == 202
job_id = queued.json()["job_id"]
assert work_once(StageContext.create(settings, llm=FakeLLM()))
status_response = client.get(
f"/internal/knowledge-graph/jobs/{job_id}",
headers={"Authorization": f"Bearer {full_token}"},
)
assert status_response.status_code == 200
assert status_response.json()["status"] == "succeeded"
assert len(status_response.json()["stages"]) == 8
answer = client.post(
"/internal/knowledge-graph/answer",
headers={"Authorization": f"Bearer {full_token}"},
json={"corpus": "api-corpus", "question": "Who created Atlas?"},
)
assert answer.status_code == 200
assert answer.json()["answer"] == "Acme created Atlas. [S1]"
def test_failed_job_can_be_repaired_and_retried(tmp_path: Path) -> None:
source = tmp_path / "retry-source.jsonl"
source.write_text(
json.dumps(
{
"schema_version": "2.0",
"external_id": "retry-record",
"corpus": "retry-corpus",
"url": "https://docs.example/retry",
"retrieved_at": "2026-10-13T18:00:00Z",
"body": "invalid schema",
}
)
+ "\n",
encoding="utf-8",
)
settings = Settings(
_env_file=None,
database_url=f"sqlite:///{tmp_path / 'retry.db'}",
auto_create_schema=True,
artifact_dir=tmp_path / "artifacts",
llm_embedding_dimensions=3,
internal_jwt_secret="dev-test-secret-at-least-32-bytes-long",
)
app = create_app(settings, llm=FakeLLM())
token = _token(
settings,
int(Permission.READ | Permission.ADD | Permission.EXECUTE),
)
headers = {"Authorization": f"Bearer {token}"}
with TestClient(app) as client:
queued = client.post(
"/internal/knowledge-graph/jobs",
headers=headers,
json={
"corpus": "retry-corpus",
"from_stage": 1,
"through_stage": 1,
"input_jsonl": str(source),
},
)
job_id = queued.json()["job_id"]
assert work_once(StageContext.create(settings, llm=FakeLLM()))
assert (
client.get(
f"/internal/knowledge-graph/jobs/{job_id}",
headers=headers,
).json()["status"]
== "failed"
)
source.write_text(
json.dumps(
{
"schema_version": "1.0",
"external_id": "retry-record",
"corpus": "retry-corpus",
"url": "https://docs.example/retry",
"retrieved_at": "2026-10-13T18:00:00Z",
"content_type": "text/plain",
"body": "Corrected source content.",
}
)
+ "\n",
encoding="utf-8",
)
retried = client.post(
f"/internal/knowledge-graph/jobs/{job_id}/retry",
headers=headers,
)
assert retried.status_code == 202
assert work_once(StageContext.create(settings, llm=FakeLLM()))
status_response = client.get(
f"/internal/knowledge-graph/jobs/{job_id}",
headers=headers,
)
assert status_response.json()["status"] == "succeeded"
assert status_response.json()["attempt"] == 2
def test_retry_resumes_after_last_successful_stage(tmp_path: Path) -> None:
source = tmp_path / "resume-source.jsonl"
source.write_text(
json.dumps(
{
"schema_version": "1.0",
"external_id": "resume-record",
"corpus": "resume-corpus",
"url": "https://docs.example/resume",
"retrieved_at": "2026-10-13T18:00:00Z",
"content_type": "text/plain",
"body": "Acme created Atlas for graph retrieval.",
}
)
+ "\n",
encoding="utf-8",
)
model_file = tmp_path / "model.json"
model_file.write_text("[]", encoding="utf-8")
settings = Settings(
_env_file=None,
database_url=f"sqlite:///{tmp_path / 'resume.db'}",
auto_create_schema=True,
artifact_dir=tmp_path / "artifacts",
llm_embedding_dimensions=3,
internal_jwt_secret="dev-test-secret-at-least-32-bytes-long",
)
app = create_app(settings, llm=FakeLLM())
token = _token(
settings,
int(Permission.READ | Permission.ADD | Permission.EXECUTE),
)
headers = {"Authorization": f"Bearer {token}"}
with TestClient(app) as client:
queued = client.post(
"/internal/knowledge-graph/jobs",
headers=headers,
json={
"corpus": "resume-corpus",
"from_stage": 1,
"through_stage": 3,
"input_jsonl": str(source),
"model_file": str(model_file),
},
)
job_id = queued.json()["job_id"]
assert work_once(StageContext.create(settings, llm=FakeLLM()))
failed = client.get(
f"/internal/knowledge-graph/jobs/{job_id}", headers=headers
).json()
assert failed["status"] == "failed"
model_file.write_text(
json.dumps(settings.model_definition()),
encoding="utf-8",
)
assert (
client.post(
f"/internal/knowledge-graph/jobs/{job_id}/retry",
headers=headers,
).status_code
== 202
)
assert work_once(StageContext.create(settings, llm=FakeLLM()))
succeeded = client.get(
f"/internal/knowledge-graph/jobs/{job_id}", headers=headers
).json()
assert succeeded["status"] == "succeeded"
stage_numbers = [row["stage"] for row in succeeded["stages"]]
assert stage_numbers.count(1) == 1
assert stage_numbers.count(2) == 1
assert stage_numbers.count(3) == 2

View File

@ -0,0 +1,69 @@
import json
from pathlib import Path
from sqlalchemy import func, select
from graphrag_platform.config import Settings
from graphrag_platform.models import Community, Edge, Node, SearchItem
from graphrag_platform.runtime import StageContext
from graphrag_platform.stage_loader import load_stage
from conftest import FakeLLM
def test_all_ten_stages_build_and_query_graph(tmp_path: Path) -> None:
input_file = tmp_path / "documents.jsonl"
input_file.write_text(
json.dumps(
{
"schema_version": "1.0",
"external_id": "atlas-doc",
"corpus": "atlas-corpus",
"url": "https://docs.example/atlas",
"retrieved_at": "2026-10-13T18:00:00Z",
"body": (
"<html><head><title>Atlas</title></head><body><main>"
"<h1>Atlas</h1><p>Acme created Atlas. Atlas provides graph "
"retrieval for knowledge applications.</p></main></body></html>"
),
"metadata": {"collector": "Web-Access", "stored_by": "Data-Store"},
}
)
+ "\n",
encoding="utf-8",
)
settings = Settings(
_env_file=None,
database_url=f"sqlite:///{tmp_path / 'graph.db'}",
auto_create_schema=True,
artifact_dir=tmp_path / "artifacts",
corpus_name="atlas-corpus",
chunk_size=300,
chunk_overlap=30,
retrieval_top_k=5,
llm_embedding_dimensions=3,
)
context = StageContext.create(settings, llm=FakeLLM())
assert load_stage(1).run(context, input_jsonl=input_file)["normalized"] == 1
assert load_stage(2).run(context)["chunks"] == 1
assert load_stage(3).run(context)["entity_types"] == 6
assert load_stage(4).run(context)["extracted_chunks"] == 1
assert load_stage(5).run(context) == {"entities": 2, "relationships": 1}
assert load_stage(6).run(context) == {"nodes": 2, "edges": 1}
assert load_stage(7).run(context)["communities"] == 1
assert load_stage(8).run(context)["indexed_items"] == 5
retrieval = load_stage(9).run(context, question="Who created Atlas?")
response = load_stage(10).run(context, question="Who created Atlas?")
assert retrieval["context"]
assert retrieval["graph_context"][0]["relationship"] == "CREATED"
assert retrieval["sources"][0]["url"] == "https://docs.example/atlas"
assert response["answer"] == "Acme created Atlas. [S1]"
assert response["sources"][0]["citation"] == "S1"
with context.storage.session() as session:
assert session.scalar(select(func.count()).select_from(Node)) == 2
assert session.scalar(select(func.count()).select_from(Edge)) == 1
assert session.scalar(select(func.count()).select_from(Community)) == 1
assert session.scalar(select(func.count()).select_from(SearchItem)) == 5

View File

@ -0,0 +1,140 @@
from __future__ import annotations
import os
from pathlib import Path
import pytest
from sqlalchemy import text
from graphrag_platform.config import Settings
from graphrag_platform.models import PipelineRun
from graphrag_platform.orchestrator import (
RunDeferredError,
claim_next_run,
execute_run,
submit_run,
)
from graphrag_platform.runtime import StageContext
from graphrag_platform.stage_loader import load_stage
from conftest import FakeLLM
POSTGRES_URL = os.getenv("TEST_POSTGRES_URL")
@pytest.mark.skipif(
not POSTGRES_URL,
reason="TEST_POSTGRES_URL is required for the PostgreSQL integration test",
)
def test_pgvector_index_and_graph_retrieval(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
settings = Settings(
_env_file=None,
database_url=POSTGRES_URL,
source_database_url=POSTGRES_URL,
source_table_name="external_documents",
artifact_dir=tmp_path / "artifacts",
corpus_name="pgvector-integration",
chunk_size=300,
chunk_overlap=30,
retrieval_top_k=5,
llm_embedding_dimensions=3,
)
monkeypatch.setenv("DATABASE_URL", POSTGRES_URL)
from alembic import command
from alembic.config import Config
alembic_config = Config("alembic.ini")
command.downgrade(alembic_config, "base")
command.upgrade(alembic_config, "head")
context = StageContext.create(settings, llm=FakeLLM())
context.storage.clear_from_stage(settings.corpus_name, 1)
with context.storage.engine.begin() as connection:
connection.execute(text("DROP TABLE IF EXISTS external_documents"))
connection.execute(
text(
"""
CREATE TABLE external_documents (
id bigserial PRIMARY KEY,
external_id text NOT NULL UNIQUE,
schema_version text NOT NULL,
corpus text,
url text NOT NULL,
body text NOT NULL,
content_type text NOT NULL,
status_code integer NOT NULL,
retrieved_at timestamptz NOT NULL,
metadata jsonb NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
)
"""
)
)
connection.execute(
text(
"""
INSERT INTO external_documents (
external_id, schema_version, corpus, url, body,
content_type, status_code, retrieved_at, metadata
) VALUES (
'atlas-doc', '1.0', 'pgvector-integration',
'https://docs.example/atlas',
'<html><head><title>Atlas</title></head><body><main><h1>Atlas</h1><p>Acme created Atlas. Atlas provides graph retrieval for knowledge applications.</p></main></body></html>',
'text/html', 200, '2026-10-13T18:00:00Z',
'{"collector":"Web-Access","stored_by":"Data-Store"}'::jsonb
)
"""
)
)
load_stage(1).run(context)
load_stage(2).run(context)
load_stage(3).run(context)
load_stage(4).run(context)
load_stage(5).run(context)
load_stage(6).run(context)
load_stage(7).run(context)
load_stage(8).run(context)
result = load_stage(9).run(context, question="Who created Atlas?")
assert result["context"]
assert result["graph_context"][0]["relationship"] == "CREATED"
assert result["sources"][0]["url"] == "https://docs.example/atlas"
with context.storage.engine.connect() as connection:
assert connection.scalar(
text("SELECT extversion FROM pg_extension WHERE extname = 'vector'")
)
index_definition = connection.scalar(
text(
"SELECT indexdef FROM pg_indexes "
"WHERE indexname = 'ix_retrieval_items_embedding_hnsw_3'"
)
)
assert index_definition is not None
assert "USING hnsw" in index_definition
run_id = submit_run(
context,
from_stage=2,
through_stage=2,
payload={},
requested_by="integration-test",
correlation_id="pg-lock-test",
permission_mask=31,
)
with context.storage.engine.connect() as lock_connection:
assert lock_connection.scalar(
text("SELECT pg_try_advisory_lock(hashtext(:key))"),
{"key": "knowledgegraph:pgvector-integration"},
)
with pytest.raises(RunDeferredError):
execute_run(context, run_id)
lock_connection.execute(
text("SELECT pg_advisory_unlock(hashtext(:key))"),
{"key": "knowledgegraph:pgvector-integration"},
)
with context.storage.session() as session:
assert session.get(PipelineRun, run_id).status == "queued"
assert claim_next_run(context) == run_id
assert execute_run(context, run_id)["2"]["chunks"] == 1

View File

@ -0,0 +1,64 @@
from __future__ import annotations
import os
import time
import pytest
from graphrag_platform.config import Settings
from graphrag_platform.models import SearchItem
from graphrag_platform.runtime import StageContext
from graphrag_platform.stage_loader import load_stage
from conftest import FakeLLM
POSTGRES_URL = os.getenv("TEST_POSTGRES_URL")
RUN_PERFORMANCE_TESTS = os.getenv("RUN_PERFORMANCE_TESTS") == "1"
@pytest.mark.performance
@pytest.mark.skipif(
not POSTGRES_URL or not RUN_PERFORMANCE_TESTS,
reason="Set TEST_POSTGRES_URL and RUN_PERFORMANCE_TESTS=1",
)
def test_vector_query_latency_at_configured_load() -> None:
item_count = int(os.getenv("PERF_VECTOR_ITEM_COUNT", "10000"))
maximum_seconds = float(os.getenv("PERF_MAX_QUERY_SECONDS", "2.0"))
settings = Settings(
_env_file=None,
database_url=POSTGRES_URL,
corpus_name="performance-corpus",
llm_embedding_dimensions=3,
retrieval_top_k=10,
)
context = StageContext.create(settings, llm=FakeLLM())
context.initialize()
with context.storage.session() as session:
session.query(SearchItem).filter_by(corpus=settings.corpus_name).delete()
session.add_all(
SearchItem(
corpus=settings.corpus_name,
item_type="benchmark",
item_key=str(index),
text=f"Benchmark graph item {index}",
source_url=None,
metadata_json={},
embedding=[
float(index % 2),
float((index // 2) % 2),
(index % 255) / 255,
],
embedding_model=FakeLLM.embedding_model,
)
for index in range(item_count)
)
context.storage.ensure_vector_index(3)
started = time.perf_counter()
result = load_stage(9).run(context, question="Atlas graph retrieval", top_k=10)
elapsed = time.perf_counter() - started
assert len(result["context"]) == 10
assert elapsed <= maximum_seconds, (
f"vector query took {elapsed:.3f}s; threshold is {maximum_seconds:.3f}s"
)

25
tests/unit/test_config.py Normal file
View File

@ -0,0 +1,25 @@
from pathlib import Path
from graphrag_platform.config import Settings
def test_settings_parse_runtime_templates(monkeypatch) -> None:
monkeypatch.setenv("ENTITY_TYPES", "Company,Product")
monkeypatch.setenv("RELATIONSHIP_TYPES", "CREATED,DEPENDS_ON")
monkeypatch.setenv("NODE_TEMPLATE", '{"name":"string"}')
settings = Settings(_env_file=None)
assert settings.entity_types == ["Company", "Product"]
assert settings.relationship_types == ["CREATED", "DEPENDS_ON"]
assert settings.node_template == {"name": "string"}
assert settings.database_url.startswith("postgresql+psycopg://")
def test_sqlite_is_available_as_explicit_local_override(tmp_path: Path) -> None:
settings = Settings(
_env_file=None,
database_url=f"sqlite:///{tmp_path / 'graph.db'}",
)
assert settings.database_url.startswith("sqlite:///")

66
tests/unit/test_llm.py Normal file
View File

@ -0,0 +1,66 @@
from __future__ import annotations
import httpx
from graphrag_platform.config import Settings
from graphrag_platform.llm import OpenAICompatibleClient
def test_llm_retries_rate_limit_and_honors_retry_after() -> None:
requests = 0
delays: list[float] = []
def handler(request: httpx.Request) -> httpx.Response:
nonlocal requests
requests += 1
if requests == 1:
return httpx.Response(
429,
headers={"Retry-After": "0.25"},
json={"error": "rate limited"},
request=request,
)
return httpx.Response(
200,
json={"choices": [{"message": {"content": "answer"}}]},
request=request,
)
settings = Settings(
_env_file=None,
llm_api_key="test-key",
llm_max_retries=2,
)
client = OpenAICompatibleClient(
settings,
transport=httpx.MockTransport(handler),
sleeper=delays.append,
jitter=lambda: 0,
)
assert client.complete_text("system", "user") == "answer"
assert requests == 2
assert delays == [0.25]
def test_llm_does_not_retry_authentication_failure() -> None:
requests = 0
def handler(request: httpx.Request) -> httpx.Response:
nonlocal requests
requests += 1
return httpx.Response(401, json={"error": "unauthorized"}, request=request)
client = OpenAICompatibleClient(
Settings(_env_file=None, llm_api_key="bad-key", llm_max_retries=3),
transport=httpx.MockTransport(handler),
sleeper=lambda _: None,
)
try:
client.complete_text("system", "user")
except httpx.HTTPStatusError:
pass
else:
raise AssertionError("Expected an HTTPStatusError")
assert requests == 1

113
tests/unit/test_pipeline.py Normal file
View File

@ -0,0 +1,113 @@
from pathlib import Path
import pytest
from sqlalchemy import select
from graphrag_platform.config import Settings
from graphrag_platform.models import Chunk, Document, Extraction, ResolvedRelationship
from graphrag_platform.runtime import StageContext
from graphrag_platform.stage_loader import load_stage
from conftest import FakeLLM
def test_chunk_overlap_must_be_smaller_than_chunk_size(tmp_path: Path) -> None:
settings = Settings(
_env_file=None,
database_url=f"sqlite:///{tmp_path / 'graph.db'}",
auto_create_schema=True,
artifact_dir=tmp_path / "artifacts",
chunk_size=200,
chunk_overlap=200,
)
stage = load_stage(2)
with pytest.raises(ValueError, match="CHUNK_OVERLAP"):
stage.chunk_text("word " * 100, size=200, overlap=200)
def test_reconciliation_keeps_cross_chunk_relationship(tmp_path: Path) -> None:
settings = Settings(
_env_file=None,
database_url=f"sqlite:///{tmp_path / 'graph.db'}",
auto_create_schema=True,
artifact_dir=tmp_path / "artifacts",
corpus_name="unit",
)
context = StageContext.create(settings, llm=FakeLLM())
context.initialize()
with context.storage.session() as session:
document = Document(
corpus="unit",
url="https://example.test/source",
text="Acme created Atlas.",
content_hash="document",
metadata_json={},
)
session.add(document)
session.flush()
first = Chunk(
document_id=document.id,
corpus="unit",
ordinal=0,
text="Acme created",
content_hash="first",
)
second = Chunk(
document_id=document.id,
corpus="unit",
ordinal=1,
text="Atlas",
content_hash="second",
)
session.add_all([first, second])
session.flush()
session.add_all(
[
Extraction(
chunk_id=first.id,
corpus="unit",
model_name="test",
payload_json={
"entities": [
{
"canonical_name": "Acme",
"entity_type": "Organization",
"aliases": [],
}
],
"relationships": [
{
"source": "Acme",
"target": "Atlas",
"relationship_type": "CREATED",
}
],
},
),
Extraction(
chunk_id=second.id,
corpus="unit",
model_name="test",
payload_json={
"entities": [
{
"canonical_name": "Atlas",
"entity_type": "Product",
"aliases": [],
}
],
"relationships": [],
},
),
]
)
result = load_stage(5).run(context)
assert result == {"entities": 2, "relationships": 1}
with context.storage.session() as session:
relationship = session.scalar(select(ResolvedRelationship))
assert relationship is not None
assert relationship.source_key == "acme"
assert relationship.target_key == "atlas"

View File

@ -0,0 +1,53 @@
from __future__ import annotations
import json
from pathlib import Path
import pytest
from graphrag_platform.sources import JsonLinesSourceReader
def test_jsonl_source_contract_preserves_nested_metadata(tmp_path: Path) -> None:
source = tmp_path / "source.jsonl"
source.write_text(
json.dumps(
{
"schema_version": "1.0",
"external_id": "record-1",
"corpus": "test",
"url": "https://example.test/article",
"body": "<main>Text, with commas and \"quotes\".</main>",
"retrieved_at": "2026-10-13T18:00:00Z",
"metadata": {"tags": ["one", "two"], "nested": {"value": 1}},
}
)
+ "\n",
encoding="utf-8",
)
records = list(JsonLinesSourceReader(source).records())
assert records[0].external_id == "record-1"
assert records[0].metadata["tags"] == ["one", "two"]
assert '"quotes"' in records[0].body
def test_jsonl_source_contract_rejects_unknown_schema(tmp_path: Path) -> None:
source = tmp_path / "source.jsonl"
source.write_text(
json.dumps(
{
"schema_version": "2.0",
"external_id": "record-1",
"url": "https://example.test/article",
"body": "text",
"retrieved_at": "2026-10-13T18:00:00Z",
}
)
+ "\n",
encoding="utf-8",
)
with pytest.raises(ValueError, match="unsupported source schema"):
list(JsonLinesSourceReader(source).records())