diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..27d2442 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,13 @@ +.env +.git +.github +.venv +__pycache__ +*.py[cod] +.pytest_cache +.coverage +artifacts +build +dist +*.egg-info +tests diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..5231978 --- /dev/null +++ b/.env.example @@ -0,0 +1,47 @@ +# Storage. Replace the host, user, password, and database for each environment. +POSTGRES_PASSWORD=knowledgegraph-dev +POSTGRES_PORT=5432 +API_PORT=8080 +DATABASE_URL=postgresql+psycopg://knowledgegraph_app:knowledgegraph-dev@localhost:5432/knowledgegraph +SOURCE_DATABASE_URL= +SOURCE_TABLE_NAME=external_documents +ANALYTICS_DATABASE_URL= +AUTO_CREATE_SCHEMA=false + +# External handoff and corpus. +CORPUS_NAME=product-knowledge + +# Runtime model templates. JSON files passed with --model-file override these values. +ENTITY_TYPES=Person,Organization,Product,Concept,Event,Location +RELATIONSHIP_TYPES=AFFILIATED_WITH,CREATED,DEPENDS_ON,MENTIONS,PART_OF,RELATED_TO +NODE_TEMPLATE={"canonical_name":"string","entity_type":"one configured entity type","description":"source-grounded string","aliases":["string"]} +EDGE_TEMPLATE={"source":"entity name","target":"entity name","relationship_type":"one configured relationship type","description":"source-grounded string"} +EXTRACTION_INSTRUCTIONS=Extract only entities and relationships explicitly supported by the supplied text. + +# OpenAI-compatible chat and embedding provider. +LLM_BASE_URL=https://api.openai.com/v1 +LLM_API_KEY=replace-with-provider-key +LLM_CHAT_MODEL=gpt-4.1-mini +LLM_EMBEDDING_MODEL=text-embedding-3-small +LLM_EMBEDDING_DIMENSIONS=1536 +LLM_TIMEOUT_SECONDS=60 +LLM_MAX_RETRIES=4 +LLM_RETRY_INITIAL_SECONDS=1 +LLM_RETRY_MAX_SECONDS=30 + +# Processing and retrieval. +CHUNK_SIZE=1200 +CHUNK_OVERLAP=150 +COMMUNITY_RESOLUTION=1.0 +RETRIEVAL_TOP_K=8 +RETRIEVAL_GRAPH_HOPS=2 +RETRIEVAL_GRAPH_LIMIT=50 +RETRIEVAL_HNSW_EF_SEARCH=100 +LOG_LEVEL=INFO +INTERNAL_JWT_SECRET=replace-with-a-dev-test-secret-at-least-32-bytes +INTERNAL_JWT_ISSUER=knowledgegraph-bff +INTERNAL_JWT_AUDIENCE=knowledgegraph-service +INTERNAL_FEATURE_NAME=KnowledgeGraph +WORKER_POLL_SECONDS=2 +WORKER_STALE_RUN_SECONDS=3600 +ARTIFACT_DIR=artifacts \ No newline at end of file diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..3207bca --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,56 @@ +name: CI + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + services: + postgres: + image: pgvector/pgvector:pg16 + env: + POSTGRES_USER: knowledgegraph_app + POSTGRES_PASSWORD: test-password + POSTGRES_DB: knowledgegraph + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U knowledgegraph_app -d knowledgegraph" + --health-interval 5s + --health-timeout 5s + --health-retries 20 + + env: + DATABASE_URL: postgresql+psycopg://knowledgegraph_app:test-password@localhost:5432/knowledgegraph + TEST_POSTGRES_URL: postgresql+psycopg://knowledgegraph_app:test-password@localhost:5432/knowledgegraph + + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + cache: pip + + - name: Install + run: python -m pip install -e ".[test]" + + - name: Compile + run: python -m compileall -q src tests migrations + + - name: Check dependencies + run: python -m pip check + + - name: Run portable tests + run: python -m pytest -q --ignore=tests/integration/test_pgvector_postgres.py + + - name: Run migration and PgVector integration + run: python -m pytest tests/integration/test_pgvector_postgres.py -q + + - name: Validate Compose + run: docker compose config --quiet + + - name: Build application image + run: docker build -f Containerfile -t knowledgegraph:test . diff --git a/.gitignore b/.gitignore index d5dca0c..255642b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,414 +1,12 @@ -# ---> VisualStudio -## Ignore Visual Studio temporary files, build results, and -## files generated by popular Visual Studio add-ons. -## -## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore - -# User-specific files -*.rsuser -*.suo -*.user -*.userosscache -*.sln.docstates - -# User-specific files (MonoDevelop/Xamarin Studio) -*.userprefs - -# Mono auto generated files -mono_crash.* - -# Build results -[Dd]ebug/ -[Dd]ebugPublic/ -[Rr]elease/ -[Rr]eleases/ -x64/ -x86/ -[Ww][Ii][Nn]32/ -[Aa][Rr][Mm]/ -[Aa][Rr][Mm]64/ -bld/ -[Bb]in/ -[Oo]bj/ -[Ll]og/ -[Ll]ogs/ - -# Visual Studio 2015/2017 cache/options directory -.vs/ -# Uncomment if you have tasks that create the project's static files in wwwroot -#wwwroot/ - -# Visual Studio 2017 auto generated files -Generated\ Files/ - -# MSTest test Results -[Tt]est[Rr]esult*/ -[Bb]uild[Ll]og.* - -# NUnit -*.VisualState.xml -TestResult.xml -nunit-*.xml - -# Build Results of an ATL Project -[Dd]ebugPS/ -[Rr]eleasePS/ -dlldata.c - -# Benchmark Results -BenchmarkDotNet.Artifacts/ - -# .NET Core -project.lock.json -project.fragment.lock.json -artifacts/ - -# ASP.NET Scaffolding -ScaffoldingReadMe.txt - -# StyleCop -StyleCopReport.xml - -# Files built by Visual Studio -*_i.c -*_p.c -*_h.h -*.ilk -*.meta -*.obj -*.iobj -*.pch -*.pdb -*.ipdb -*.pgc -*.pgd -*.rsp -*.sbr -*.tlb -*.tli -*.tlh -*.tmp -*.tmp_proj -*_wpftmp.csproj -*.log -*.tlog -*.vspscc -*.vssscc -.builds -*.pidb -*.svclog -*.scc - -# Chutzpah Test files -_Chutzpah* - -# Visual C++ cache files -ipch/ -*.aps -*.ncb -*.opendb -*.opensdf -*.sdf -*.cachefile -*.VC.db -*.VC.VC.opendb - -# Visual Studio profiler -*.psess -*.vsp -*.vspx -*.sap - -# Visual Studio Trace Files -*.e2e - -# TFS 2012 Local Workspace -$tf/ - -# Guidance Automation Toolkit -*.gpState - -# ReSharper is a .NET coding add-in -_ReSharper*/ -*.[Rr]e[Ss]harper -*.DotSettings.user - -# TeamCity is a build add-in -_TeamCity* - -# DotCover is a Code Coverage Tool -*.dotCover - -# AxoCover is a Code Coverage Tool -.axoCover/* -!.axoCover/settings.json - -# Coverlet is a free, cross platform Code Coverage Tool -coverage*.json -coverage*.xml -coverage*.info - -# Visual Studio code coverage results -*.coverage -*.coveragexml - -# NCrunch -_NCrunch_* -.*crunch*.local.xml -nCrunchTemp_* - -# MightyMoose -*.mm.* -AutoTest.Net/ - -# Web workbench (sass) -.sass-cache/ - -# Installshield output folder -[Ee]xpress/ - -# DocProject is a documentation generator add-in -DocProject/buildhelp/ -DocProject/Help/*.HxT -DocProject/Help/*.HxC -DocProject/Help/*.hhc -DocProject/Help/*.hhk -DocProject/Help/*.hhp -DocProject/Help/Html2 -DocProject/Help/html - -# Click-Once directory -publish/ - -# Publish Web Output -*.[Pp]ublish.xml -*.azurePubxml -# Note: Comment the next line if you want to checkin your web deploy settings, -# but database connection strings (with potential passwords) will be unencrypted -*.pubxml -*.publishproj - -# Microsoft Azure Web App publish settings. Comment the next line if you want to -# checkin your Azure Web App publish settings, but sensitive information contained -# in these scripts will be unencrypted -PublishScripts/ - -# NuGet Packages -*.nupkg -# NuGet Symbol Packages -*.snupkg -# The packages folder can be ignored because of Package Restore -**/[Pp]ackages/* -# except build/, which is used as an MSBuild target. -!**/[Pp]ackages/build/ -# Uncomment if necessary however generally it will be regenerated when needed -#!**/[Pp]ackages/repositories.config -# NuGet v3's project.json files produces more ignorable files -*.nuget.props -*.nuget.targets - -# Microsoft Azure Build Output -csx/ -*.build.csdef - -# Microsoft Azure Emulator -ecf/ -rcf/ - -# Windows Store app package directories and files -AppPackages/ -BundleArtifacts/ -Package.StoreAssociation.xml -_pkginfo.txt -*.appx -*.appxbundle -*.appxupload - -# Visual Studio cache files -# files ending in .cache can be ignored -*.[Cc]ache -# but keep track of directories ending in .cache -!?*.[Cc]ache/ - -# Others -ClientBin/ -~$* -*~ -*.dbmdl -*.dbproj.schemaview -*.jfm -*.pfx -*.publishsettings -orleans.codegen.cs - -# Including strong name files can present a security risk -# (https://github.com/github/gitignore/pull/2483#issue-259490424) -#*.snk - -# Since there are multiple workflows, uncomment next line to ignore bower_components -# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622) -#bower_components/ - -# RIA/Silverlight projects -Generated_Code/ - -# Backup & report files from converting an old project file -# to a newer Visual Studio version. Backup files are not needed, -# because we have git ;-) -_UpgradeReport_Files/ -Backup*/ -UpgradeLog*.XML -UpgradeLog*.htm -ServiceFabricBackup/ -*.rptproj.bak - -# SQL Server files -*.mdf -*.ldf -*.ndf - -# Business Intelligence projects -*.rdl.data -*.bim.layout -*.bim_*.settings -*.rptproj.rsuser -*- [Bb]ackup.rdl -*- [Bb]ackup ([0-9]).rdl -*- [Bb]ackup ([0-9][0-9]).rdl - -# Microsoft Fakes -FakesAssemblies/ - -# GhostDoc plugin setting file -*.GhostDoc.xml - -# Node.js Tools for Visual Studio -.ntvs_analysis.dat -node_modules/ - -# Visual Studio 6 build log -*.plg - -# Visual Studio 6 workspace options file -*.opt - -# Visual Studio 6 auto-generated workspace file (contains which files were open etc.) -*.vbw - -# Visual Studio 6 auto-generated project file (contains which files were open etc.) -*.vbp - -# Visual Studio 6 workspace and project file (working project files containing files to include in project) -*.dsw -*.dsp - -# Visual Studio 6 technical files -*.ncb -*.aps - -# Visual Studio LightSwitch build output -**/*.HTMLClient/GeneratedArtifacts -**/*.DesktopClient/GeneratedArtifacts -**/*.DesktopClient/ModelManifest.xml -**/*.Server/GeneratedArtifacts -**/*.Server/ModelManifest.xml -_Pvt_Extensions - -# Paket dependency manager -.paket/paket.exe -paket-files/ - -# FAKE - F# Make -.fake/ - -# CodeRush personal settings -.cr/personal - -# Python Tools for Visual Studio (PTVS) +.venv/ __pycache__/ -*.pyc - -# Cake - Uncomment if you are using it -# tools/** -# !tools/packages.config - -# Tabs Studio -*.tss - -# Telerik's JustMock configuration file -*.jmconfig - -# BizTalk build output -*.btp.cs -*.btm.cs -*.odx.cs -*.xsd.cs - -# OpenCover UI analysis results -OpenCover/ - -# Azure Stream Analytics local run output -ASALocalRun/ - -# MSBuild Binary and Structured Log -*.binlog - -# NVidia Nsight GPU debugger configuration file -*.nvuser - -# MFractors (Xamarin productivity tool) working folder -.mfractor/ - -# Local History for Visual Studio -.localhistory/ - -# Visual Studio History (VSHistory) files -.vshistory/ - -# BeatPulse healthcheck temp database -healthchecksdb - -# Backup folder for Package Reference Convert tool in Visual Studio 2017 -MigrationBackup/ - -# Ionide (cross platform F# VS Code tools) working folder -.ionide/ - -# Fody - auto-generated XML schema -FodyWeavers.xsd - -# VS Code files for those working on multiple tools -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json -*.code-workspace - -# Local History for Visual Studio Code -.history/ - -# Windows Installer files from build outputs -*.cab -*.msi -*.msix -*.msm -*.msp - -# JetBrains Rider -*.sln.iml - -# ---> VisualStudioCode -.vscode/* -!.vscode/settings.json -!.vscode/tasks.json -!.vscode/launch.json -!.vscode/extensions.json -!.vscode/*.code-snippets - -# Local History for Visual Studio Code -.history/ - -# Built Visual Studio Code Extensions -*.vsix - +*.py[cod] +.pytest_cache/ +.coverage +htmlcov/ +.env +*.db +artifacts/ +dist/ +build/ +*.egg-info/ \ No newline at end of file diff --git a/Containerfile b/Containerfile new file mode 100644 index 0000000..3de9289 --- /dev/null +++ b/Containerfile @@ -0,0 +1,26 @@ +FROM python:3.12-slim AS runtime + +ENV PYTHONDONTWRITEBYTECODE=1 \ + PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +RUN groupadd --system knowledgegraph \ + && useradd --system --gid knowledgegraph --create-home knowledgegraph + +WORKDIR /app + +COPY pyproject.toml README.md alembic.ini ./ +COPY migrations ./migrations +COPY src ./src + +RUN python -m pip install --upgrade pip \ + && python -m pip install . + +RUN mkdir -p /app/artifacts \ + && chown -R knowledgegraph:knowledgegraph /app + +USER knowledgegraph + +EXPOSE 8080 + +CMD ["uvicorn", "graphrag_platform.api:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/README.md b/README.md index 978e4d4..574e893 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,642 @@ -# RAG-POC +# KnowledgeGraph GraphRAG +An internal, restartable GraphRAG pipeline that normalizes externally collected +content, extracts and reconciles knowledge, builds a graph, indexes graph and source +evidence with PostgreSQL/PgVector, retrieves graph-aware context, and generates +source-cited answers. + +The collection process is outside this project. Existing **Web-Access MCP** and +**Data-Store MCP** processes discover and persist source material. This project +starts by validating and normalizing their versioned handoff. + +## Architecture + +```text +Web-Access MCP + Data-Store MCP + | + | versioned JSONL or PostgreSQL handoff + v +01 normalize and parse + v +02 clean, deduplicate, and chunk + v +03 define knowledge model + v +04 extract entities and relationships + v +05 reconcile aliases and duplicates + v +06 materialize graph nodes and edges + v +07 detect and summarize communities + v +08 build PgVector HNSW retrieval index + v +09 retrieve vector + graph + source context + v +10 generate a cited answer +``` + +PostgreSQL is the canonical checkpoint store. Every completed build stage persists +its output, so later stages can be rebuilt without repeating valid upstream work. +PgVector stores embeddings and performs server-side HNSW cosine search. Indexed +relational edge tables provide bounded graph expansion. + +The deployment consists of: + +- PostgreSQL 16 with PgVector +- Alembic migration job +- Internal FastAPI service +- PostgreSQL-backed asynchronous worker +- Existing BFF as the policy enforcement point + +The API is internal only and is not intended for direct customer access. + +## Repository layout + +| Path | Purpose | +|---|---| +| [`src/01_normalize_content.py`](src/01_normalize_content.py) through [`src/10_generate_answer.py`](src/10_generate_answer.py) | Complete numbered stage implementations | +| [`src/graphrag_platform`](src/graphrag_platform) | Configuration, models, storage, orchestration, API, auth, worker, and provider clients | +| [`migrations`](migrations) | Alembic database migrations | +| [`tests`](tests) | Unit, API/worker, pipeline, migration, and real PgVector integration tests | +| [`compose.yaml`](compose.yaml) | Reproducible PostgreSQL, migration, API, and worker environment | +| [`Containerfile`](Containerfile) | Non-root application image | +| [`.env.example`](.env.example) | Runtime configuration template | + +## Prerequisites + +- Python 3.11 or newer +- Podman 5 or newer +- `podman machine` running on Windows +- An OpenAI-compatible chat and embeddings endpoint +- External Web-Access MCP/Data-Store MCP collection process + +## Quick start with Compose + +Create the local environment: + +```powershell +Copy-Item .env.example .env +``` + +Edit `.env` and set at minimum: + +```dotenv +POSTGRES_PASSWORD=choose-a-local-password +DATABASE_URL=postgresql+psycopg://knowledgegraph_app:choose-a-local-password@localhost:5432/knowledgegraph + +LLM_BASE_URL=https://api.openai.com/v1 +LLM_API_KEY=replace-with-provider-key +LLM_CHAT_MODEL=gpt-4.1-mini +LLM_EMBEDDING_MODEL=text-embedding-3-small +LLM_EMBEDDING_DIMENSIONS=1536 + +INTERNAL_JWT_SECRET=replace-with-at-least-32-random-bytes +``` + +`POSTGRES_PASSWORD` and the password in `DATABASE_URL` must match. Compose replaces +the database host with `postgres` inside application containers. + +Start the complete environment: + +```powershell +podman machine start +podman compose up --detach --build +``` + +Compose: + +1. Starts `docker.io/pgvector/pgvector:pg16`. +2. Waits for PostgreSQL readiness. +3. Runs `alembic upgrade head`. +4. Starts the API and worker. +5. Waits for API dependency readiness. + +Verify it: + +```powershell +podman compose ps +Invoke-RestMethod http://127.0.0.1:8080/internal/knowledge-graph/health +Invoke-RestMethod http://127.0.0.1:8080/internal/knowledge-graph/ready +``` + +View logs: + +```powershell +podman compose logs --follow api worker +``` + +Stop services while retaining PostgreSQL data: + +```powershell +podman compose down +``` + +Delete services and all local database data: + +```powershell +podman compose down --volumes +``` + +Volume removal is destructive. + +## External source handoff + +### JSON Lines contract + +Each UTF-8 JSONL line must satisfy schema version `1.0`: + +```json +{ + "schema_version": "1.0", + "external_id": "stable-source-record-id", + "corpus": "product-knowledge", + "url": "https://docs.example.com/article", + "body": "
Article text
", + "content_type": "text/html", + "status_code": 200, + "retrieved_at": "2026-10-13T18:00:00Z", + "metadata": { + "collector": "Web-Access", + "stored_by": "Data-Store" + } +} +``` + +Required fields: + +- `schema_version`: currently `1.0` +- `external_id`: stable and unique within a corpus +- `url` +- `body` +- `retrieved_at`: ISO 8601 timestamp with offset + +Optional/defaulted fields: + +- `corpus`: when provided, it must match the selected runtime corpus +- `content_type`: defaults to `text/html` +- `status_code`: defaults to `200` +- `metadata`: defaults to an empty object + +Unknown fields and unsupported schema versions are rejected. Invalid records include +the file and line number in the error. + +The template is [`src/templates/mcp_documents.jsonl`](src/templates/mcp_documents.jsonl). +A larger acceptance corpus is +[`tests/fixtures/external_documents.jsonl`](tests/fixtures/external_documents.jsonl). + +### PostgreSQL source contract + +Set `SOURCE_DATABASE_URL` and `SOURCE_TABLE_NAME` to read the same contract from an +external PostgreSQL table: + +```dotenv +SOURCE_DATABASE_URL=postgresql+psycopg://source_reader:password@source-host:5432/source_database +SOURCE_TABLE_NAME=external_documents +``` + +Compatible table: + +```sql +CREATE TABLE external_documents ( + external_id text PRIMARY KEY, + schema_version text NOT NULL DEFAULT '1.0', + corpus text, + url text NOT NULL, + body text NOT NULL, + content_type text NOT NULL DEFAULT 'text/html', + status_code integer NOT NULL DEFAULT 200, + retrieved_at timestamptz NOT NULL, + metadata jsonb NOT NULL DEFAULT '{}'::jsonb +); +``` + +Additional source-table columns are ignored. The application account should receive +read-only access to the source table. + +Provide either `--input-jsonl` or `SOURCE_DATABASE_URL`, not both. + +## Durable checkpoints + +Build stages persist these boundaries: + +```text +raw_documents + documents + -> chunks + -> knowledge_models + -> extractions + -> resolved_entities + resolved_relationships + -> nodes + edges + -> communities + -> retrieval_items +``` + +`pipeline_runs` and `stage_runs` separately record: + +- queued/running/succeeded/failed state +- selected stage range +- attempt +- configuration fingerprint +- caller and correlation ID +- stage inputs and outputs +- timestamps and error details + +The worker requeues runs abandoned beyond `WORKER_STALE_RUN_SECONDS`. Set this value +above the expected maximum duration of one stage. + +PostgreSQL advisory locks allow only one active build for a corpus. If another +worker claims a job for that corpus, the job is returned to the queue instead of +executing conflicting writes. + +DataFrames are not stage checkpoints. A Pandas or Polars DataFrame may be used by a +future adapter for one bounded transformation, but every completed result must be +persisted before the next stage starts. + +## Database migrations + +Runtime services verify the migrated schema and do not silently mutate it. + +Apply migrations: + +```powershell +alembic upgrade head +``` + +Inspect the current revision: + +```powershell +alembic current +alembic history +``` + +Create a migration after changing SQLAlchemy models: + +```powershell +alembic revision --autogenerate -m "describe schema change" +``` + +Review generated migrations before applying them. Compose runs upgrades through the +one-shot `migrate` service before starting API and worker services. + +`AUTO_CREATE_SCHEMA=true` exists only for isolated tests. Do not enable it in shared +dev, test, or production environments. + +## Run stages directly + +Install a local editable environment: + +```powershell +py -3.12 -m venv .venv +.\.venv\Scripts\Activate.ps1 +python -m pip install --upgrade pip +python -m pip install -e ".[test]" +``` + +Run migrations before the first command: + +```powershell +alembic upgrade head +``` + +### 1. Normalize external content + +```powershell +python .\src\01_normalize_content.py ` + --corpus product-knowledge ` + --input-jsonl .\artifacts\mcp-documents.jsonl +``` + +This validates the external handoff, stores raw provenance, runs Trafilatura for +HTML, and persists normalized document records. + +### 2. Clean, deduplicate, and chunk + +```powershell +python .\src\02_clean_content.py --corpus product-knowledge +``` + +### 3. Define the knowledge model + +```powershell +python .\src\03_define_knowledge_model.py ` + --corpus product-knowledge ` + --model-file .\src\templates\knowledge_model.json +``` + +### 4. Extract entities and relationships + +```powershell +python .\src\04_extract_knowledge.py --corpus product-knowledge +``` + +LLM calls occur outside long database transactions. Each validated chunk result is +committed independently. + +### 5. Reconcile knowledge + +```powershell +python .\src\05_reconcile_knowledge.py --corpus product-knowledge +``` + +### 6. Build graph nodes and edges + +```powershell +python .\src\06_build_knowledge_graph.py --corpus product-knowledge +``` + +### 7. Build communities + +```powershell +python .\src\07_build_communities.py --corpus product-knowledge +``` + +### 8. Build the PgVector index + +```powershell +python .\src\08_build_retrieval_index.py --corpus product-knowledge +``` + +This embeds chunks, nodes, edges, and community summaries, validates dimensions, +stores vectors, and creates a dimension-specific HNSW cosine index. + +### 9. Retrieve GraphRAG context + +```powershell +python .\src\09_query_graphrag.py ` + --corpus product-knowledge ` + --question "Who created Atlas?" ` + --top-k 8 +``` + +The result contains vector `context`, expanded `graph_context`, and traceable +`sources`. + +### 10. Generate a cited answer + +```powershell +python .\src\10_generate_answer.py ` + --corpus product-knowledge ` + --question "Who created Atlas?" +``` + +## Run a durable build + +Run stages 1 through 8 under one durable pipeline record: + +```powershell +knowledgegraph run ` + --corpus product-knowledge ` + --from-stage 1 ` + --through-stage 8 ` + --input-jsonl .\artifacts\mcp-documents.jsonl ` + --model-file .\src\templates\knowledge_model.json ` + --requested-by local-developer +``` + +Use `--from-stage` to rebuild only downstream state. The `--reset` option deletes +the selected stage and downstream derived corpus data before execution. + +## BFF authorization contract + +The existing BFF authenticates callers and evaluates the `KnowledgeGraph` feature. +Independent permission bits are: + +```text +Read = 1 +Add = 2 +Edit = 4 +Delete = 8 +Execute = 16 +``` + +| Operation | Required mask | +|---|---:| +| Read job/source/retrieval information | `Read` (`1`) | +| Submit a stage-1 source job | `Add | Execute` (`18`) | +| Run downstream build stages | `Execute` (`16`) | +| Query or answer | `Read | Execute` (`17`) | +| Retry a failed job | `Execute` (`16`) | + +The BFF forwards a signed internal JWT. Required claims: + +```json +{ + "iss": "knowledgegraph-bff", + "aud": "knowledgegraph-service", + "sub": "internal-caller-id", + "iat": 1791930000, + "exp": 1791930600, + "jti": "correlation-or-request-id", + "features": { + "KnowledgeGraph": 31 + } +} +``` + +Dev/test uses HS256 and `INTERNAL_JWT_SECRET`. Production deployments can replace +this verifier with the BFF's workload identity, mTLS, or asymmetric/JWKS contract. +Never trust unsigned permission headers. + +## Internal API + +| Endpoint | Purpose | +|---|---| +| `GET /internal/knowledge-graph/health` | Process liveness | +| `GET /internal/knowledge-graph/ready` | Database and PgVector readiness | +| `POST /internal/knowledge-graph/jobs` | Queue stages 1–8 | +| `GET /internal/knowledge-graph/jobs/{jobId}` | Read durable job/stage status | +| `POST /internal/knowledge-graph/jobs/{jobId}/retry` | Requeue a failed job | +| `POST /internal/knowledge-graph/query` | Synchronous bounded retrieval | +| `POST /internal/knowledge-graph/answer` | Synchronous cited answer | + +Example job request: + +```json +{ + "corpus": "product-knowledge", + "from_stage": 1, + "through_stage": 8, + "input_jsonl": "/app/artifacts/mcp-documents.jsonl", + "model_file": "/app/src/templates/knowledge_model.json", + "reset": false +} +``` + +Compose bind-mounts local [`artifacts`](artifacts) at `/app/artifacts`, allowing the +worker to read JSONL handoffs placed there. In deployed environments, prefer the +PostgreSQL source contract or a shared managed file volume. + +Long-running stages return `202 Accepted` with a job ID. Query and answer requests +remain synchronous and bounded. + +## Retry and logging behavior + +Model-provider calls retry: + +- HTTP 429 +- HTTP 500, 502, 503, and 504 +- connection failures +- timeouts + +Retries honor numeric `Retry-After`, otherwise use bounded exponential backoff with +jitter. Authentication and validation errors are not retried. + +Logs are emitted as one JSON object per line and include event names, run IDs, +corpus/stage context where available, attempts, status, and error details. Secrets +are not logged. + +## Tests + +Portable tests: + +```powershell +pytest --ignore=.\tests\integration\test_pgvector_postgres.py +``` + +Real PostgreSQL/PgVector migration and retrieval test: + +```powershell +$env:TEST_POSTGRES_URL = "postgresql+psycopg://knowledgegraph_app:password@localhost:5432/knowledgegraph" +pytest .\tests\integration\test_pgvector_postgres.py +Remove-Item Env:TEST_POSTGRES_URL +``` + +The suite covers: + +- Versioned source validation +- Nested metadata and quoted/comma-containing content +- Configuration parsing +- Chunk validation +- Cross-chunk relationship reconciliation +- Provider retry policy +- Full ten-stage behavior +- Signed BFF authorization and permission failures +- Durable API job and worker execution +- Alembic migration against PostgreSQL +- PgVector extension and HNSW index creation +- Server-side vector retrieval, graph expansion, and citations + +Opt-in vector load test: + +```powershell +$env:TEST_POSTGRES_URL = "postgresql+psycopg://knowledgegraph_app:password@localhost:5432/knowledgegraph" +$env:RUN_PERFORMANCE_TESTS = "1" +$env:PERF_VECTOR_ITEM_COUNT = "10000" +$env:PERF_MAX_QUERY_SECONDS = "2.0" +pytest .\tests\performance\test_vector_query_load.py +``` + +Set the item count and latency threshold to the capacity target for the dev/test +environment. Performance checks are excluded by default because machine resources +vary. + +CI is defined in [`.github/workflows/ci.yml`](.github/workflows/ci.yml) and compiles, +tests, validates Compose, exercises real PgVector, and builds the application image. + +## Persistent format policy + +- PostgreSQL is the canonical operational store. +- JSON Lines is the preferred file handoff/checkpoint format. +- CSV is allowed only for explicitly defined flat import/export views. + +CSV safely supports commas, quotes, and line breaks when written by a compliant CSV +library. It remains unsuitable as the canonical graph format because aliases, +metadata, source ID arrays, embeddings, and graph properties are nested and typed. +Never construct CSV by joining strings with commas. + +## RAG database alternatives + +### PostgreSQL with PgVector + +Implemented default. Choose it for SQL transactions, colocated provenance and +vectors, self-managed deployment, and tunable HNSW retrieval. Graph traversal uses +indexed relational edges rather than a native property-graph language. + +Official reference: [PgVector](https://github.com/pgvector/pgvector). + +### Azure Cosmos DB for NoSQL + +Choose it for managed/serverless document storage, vector indexing, and first-party +vector plus BM25 hybrid search. Cosmos DB's Gremlin API is separate from the NoSQL +vector feature; use document adjacency or a separate graph store. + +References: + +- [Cosmos DB vector search](https://learn.microsoft.com/azure/cosmos-db/vector-search) +- [Cosmos DB hybrid search](https://learn.microsoft.com/azure/cosmos-db/gen-ai/hybrid-search) +- [Cosmos DB Gremlin API](https://learn.microsoft.com/azure/cosmos-db/gremlin/overview) + +### Neo4j + +Choose it when graph topology, paths, and multi-hop traversal are primary retrieval +signals. Neo4j combines native property graphs, Cypher, vector indexes, and +graph-aware hybrid retrieval. It requires a dedicated backend adapter; changing +`DATABASE_URL` is not sufficient. Confirm version and edition requirements for the +selected vector and unified-search features. + +References: + +- [Neo4j vector indexes](https://neo4j.com/docs/cypher-manual/current/indexes/search-indexes/vector-indexes/) +- [Neo4j hybrid search](https://neo4j.com/developer/genai-ecosystem/hybrid-search/) + +Other credible options: + +- [Amazon Neptune Analytics](https://docs.aws.amazon.com/neptune-analytics/latest/userguide/what-is-neptune-analytics.html) + for AWS-managed graph/vector analytics +- [Azure AI Search](https://learn.microsoft.com/azure/search/hybrid-search-overview) + as a secondary managed hybrid index without a native graph model + +## Operational guidance + +- Keep the API and worker on internal networks accessible only to the BFF and + operators. +- Replace all dev secrets before shared deployment. +- Use a separate PostgreSQL role for migrations and a least-privilege runtime role. +- Back up PostgreSQL before destructive resets or migrations. +- Monitor queued/running/failed run counts, stage duration, LLM retries, database + pool usage, vector latency, and retrieval result counts. +- Set `WORKER_STALE_RUN_SECONDS` above the longest expected single stage. +- Rebuild stage 8 after changing embedding model or dimensions. +- Maintain a golden question/citation set to detect retrieval-quality regressions. + +## Troubleshooting + +### Database migrations are required + +```powershell +alembic current +alembic upgrade head +``` + +### `LLM_API_KEY is required` + +Set `LLM_API_KEY` in `.env` and restart API/worker containers. + +### Embedding dimension mismatch + +Set `LLM_EMBEDDING_DIMENSIONS` to the exact provider output dimension, then reset +and rerun stage 8. + +### Source handoff rejected + +Validate every JSONL line against schema version `1.0`. Confirm `corpus`, +`external_id`, URL, body, and `retrieved_at`. + +### Job remains queued + +```powershell +podman compose ps worker +podman compose logs --tail 100 worker +``` + +### PostgreSQL/PgVector is not ready + +```powershell +podman compose ps postgres +podman compose logs --tail 100 postgres migrate +``` + +### Port conflict + +Change `POSTGRES_PORT` or `API_PORT` in `.env`, then recreate the Compose stack. diff --git a/alembic.ini b/alembic.ini new file mode 100644 index 0000000..1f3d3e9 --- /dev/null +++ b/alembic.ini @@ -0,0 +1,38 @@ +[alembic] +script_location = migrations +prepend_sys_path = . +path_separator = os + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console +qualname = + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/compose.yaml b/compose.yaml new file mode 100644 index 0000000..5ffe829 --- /dev/null +++ b/compose.yaml @@ -0,0 +1,99 @@ +name: knowledgegraph + +services: + postgres: + image: docker.io/pgvector/pgvector:pg16 + environment: + POSTGRES_USER: knowledgegraph_app + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-knowledgegraph-dev} + POSTGRES_DB: knowledgegraph + ports: + - "127.0.0.1:${POSTGRES_PORT:-5432}:5432" + volumes: + - knowledgegraph-postgres-data:/var/lib/postgresql/data + healthcheck: + test: + [ + "CMD-SHELL", + "pg_isready -U knowledgegraph_app -d knowledgegraph", + ] + interval: 5s + timeout: 5s + retries: 20 + restart: unless-stopped + + migrate: + build: + context: . + dockerfile: Containerfile + command: ["alembic", "upgrade", "head"] + environment: &application-environment + DATABASE_URL: postgresql+psycopg://knowledgegraph_app:${POSTGRES_PASSWORD:-knowledgegraph-dev}@postgres:5432/knowledgegraph + SOURCE_DATABASE_URL: ${SOURCE_DATABASE_URL:-} + SOURCE_TABLE_NAME: ${SOURCE_TABLE_NAME:-external_documents} + AUTO_CREATE_SCHEMA: "false" + CORPUS_NAME: ${CORPUS_NAME:-default-corpus} + LLM_BASE_URL: ${LLM_BASE_URL:-https://api.openai.com/v1} + LLM_API_KEY: ${LLM_API_KEY:-} + LLM_CHAT_MODEL: ${LLM_CHAT_MODEL:-gpt-4.1-mini} + LLM_EMBEDDING_MODEL: ${LLM_EMBEDDING_MODEL:-text-embedding-3-small} + LLM_EMBEDDING_DIMENSIONS: ${LLM_EMBEDDING_DIMENSIONS:-1536} + INTERNAL_JWT_SECRET: ${INTERNAL_JWT_SECRET:-knowledgegraph-dev-secret-change-this-32} + INTERNAL_JWT_ISSUER: ${INTERNAL_JWT_ISSUER:-knowledgegraph-bff} + INTERNAL_JWT_AUDIENCE: ${INTERNAL_JWT_AUDIENCE:-knowledgegraph-service} + INTERNAL_FEATURE_NAME: ${INTERNAL_FEATURE_NAME:-KnowledgeGraph} + LOG_LEVEL: ${LOG_LEVEL:-INFO} + depends_on: + postgres: + condition: service_healthy + restart: "no" + + api: + build: + context: . + dockerfile: Containerfile + command: + [ + "uvicorn", + "graphrag_platform.api:app", + "--host", + "0.0.0.0", + "--port", + "8080", + ] + environment: *application-environment + ports: + - "127.0.0.1:${API_PORT:-8080}:8080" + volumes: + - ./artifacts:/app/artifacts + depends_on: + migrate: + condition: service_completed_successfully + healthcheck: + test: + [ + "CMD", + "python", + "-c", + "import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/internal/knowledge-graph/ready', timeout=5)", + ] + interval: 10s + timeout: 5s + retries: 12 + restart: unless-stopped + + worker: + build: + context: . + dockerfile: Containerfile + command: ["knowledgegraph-worker"] + environment: *application-environment + volumes: + - ./artifacts:/app/artifacts + depends_on: + migrate: + condition: service_completed_successfully + restart: unless-stopped + +volumes: + knowledgegraph-postgres-data: diff --git a/migrations/env.py b/migrations/env.py new file mode 100644 index 0000000..8cc2451 --- /dev/null +++ b/migrations/env.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from logging.config import fileConfig + +from alembic import context +from sqlalchemy import engine_from_config, pool + +from graphrag_platform.config import Settings +from graphrag_platform.models import Base + +config = context.config +if config.config_file_name: + fileConfig(config.config_file_name) + +config.set_main_option("sqlalchemy.url", Settings().database_url) +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + context.configure( + url=config.get_main_option("sqlalchemy.url"), + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +def run_migrations_online() -> None: + connectable = engine_from_config( + config.get_section(config.config_ini_section, {}), + prefix="sqlalchemy.", + poolclass=pool.NullPool, + ) + with connectable.connect() as connection: + context.configure( + connection=connection, + target_metadata=target_metadata, + compare_type=True, + ) + with context.begin_transaction(): + context.run_migrations() + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/migrations/script.py.mako b/migrations/script.py.mako new file mode 100644 index 0000000..590f5b3 --- /dev/null +++ b/migrations/script.py.mako @@ -0,0 +1,24 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/migrations/versions/20261013_0001_initial.py b/migrations/versions/20261013_0001_initial.py new file mode 100644 index 0000000..61eafbb --- /dev/null +++ b/migrations/versions/20261013_0001_initial.py @@ -0,0 +1,25 @@ +"""Create the initial KnowledgeGraph schema. + +Revision ID: 20261013_0001 +Revises: +Create Date: 2026-10-13 +""" + +from alembic import op + +from graphrag_platform.models import Base + +revision = "20261013_0001" +down_revision = None +branch_labels = None +depends_on = None + + +def upgrade() -> None: + connection = op.get_bind() + connection.exec_driver_sql("CREATE EXTENSION IF NOT EXISTS vector") + Base.metadata.create_all(bind=connection) + + +def downgrade() -> None: + Base.metadata.drop_all(bind=op.get_bind()) diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..15147dd --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,60 @@ +[build-system] +requires = ["hatchling>=1.25"] +build-backend = "hatchling.build" + +[project] +name = "knowledgegraph-graphrag" +version = "0.1.0" +description = "Production-oriented KnowledgeGraph ingestion, indexing, retrieval, and answer pipeline" +readme = "README.md" +requires-python = ">=3.11" +dependencies = [ + "alembic>=1.13,<2", + "fastapi>=0.115,<1", + "httpx>=0.27,<1", + "networkx>=3.3,<4", + "pgvector>=0.3,<1", + "PyJWT>=2.9,<3", + "pydantic>=2.8,<3", + "pydantic-settings>=2.4,<3", + "psycopg[binary]>=3.2,<4", + "sqlalchemy>=2.0,<3", + "trafilatura>=1.12,<3", + "uvicorn[standard]>=0.32,<1", +] + +[project.optional-dependencies] +test = [ + "pytest>=8.3,<9", + "pytest-cov>=5,<7", +] + +[project.scripts] +knowledgegraph = "graphrag_platform.cli:main_all" +knowledgegraph-worker = "graphrag_platform.worker:main" + +[tool.hatch.build.targets.wheel] +packages = [ + "src/graphrag_platform", + "src/01_normalize_content.py", + "src/02_clean_content.py", + "src/03_define_knowledge_model.py", + "src/04_extract_knowledge.py", + "src/05_reconcile_knowledge.py", + "src/06_build_knowledge_graph.py", + "src/07_build_communities.py", + "src/08_build_retrieval_index.py", + "src/09_query_graphrag.py", + "src/10_generate_answer.py", +] + +[tool.pytest.ini_options] +addopts = "-ra" +testpaths = ["tests"] +markers = [ + "performance: opt-in performance and load checks", +] + +[tool.coverage.run] +source = ["graphrag_platform"] +branch = true \ No newline at end of file diff --git a/src/01_normalize_content.py b/src/01_normalize_content.py new file mode 100644 index 0000000..440c7ea --- /dev/null +++ b/src/01_normalize_content.py @@ -0,0 +1,128 @@ +from __future__ import annotations + +import argparse +import hashlib +import json +from pathlib import Path + +import trafilatura + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import Document, RawDocument +from graphrag_platform.runtime import StageContext +from graphrag_platform.sources import SourceRecord, source_reader + + +def content_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def extract_document(record: SourceRecord) -> dict[str, object]: + if "html" not in record.content_type.casefold(): + text = record.body.strip() + if not text: + raise ValueError(f"source content is empty for {record.url}") + return { + "title": record.metadata.get("title"), + "author": record.metadata.get("author"), + "date": record.metadata.get("published_at"), + "hostname": None, + "sitename": None, + "text": text, + } + extracted = trafilatura.extract( + record.body, + url=str(record.url), + output_format="json", + with_metadata=True, + include_comments=False, + include_tables=True, + favor_precision=True, + ) + if not extracted: + raise ValueError(f"could not extract article content from {record.url}") + payload = json.loads(extracted) + if not str(payload.get("text") or "").strip(): + raise ValueError(f"extracted content is empty for {record.url}") + return payload + + +def run( + context: StageContext, + *, + input_jsonl: Path | None = None, +) -> dict[str, int]: + context.initialize() + reader = source_reader( + input_jsonl=input_jsonl, + source_database_url=context.settings.source_database_url, + source_table_name=context.settings.source_table_name, + ) + normalized = 0 + with context.storage.session() as session: + for record in reader.records(): + if record.corpus and record.corpus != context.settings.corpus_name: + raise ValueError( + f"record {record.external_id} targets corpus {record.corpus}, " + f"not {context.settings.corpus_name}" + ) + payload = extract_document(record) + text = str(payload["text"]).strip() + metadata = dict(record.metadata) + metadata.update( + { + "external_id": record.external_id, + "source_schema_version": record.schema_version, + "retrieved_at": record.retrieved_at.isoformat(), + "hostname": payload.get("hostname"), + "sitename": payload.get("sitename"), + } + ) + raw = context.storage.upsert( + session, + RawDocument, + { + "corpus": context.settings.corpus_name, + "external_id": record.external_id, + }, + { + "source_schema_version": record.schema_version, + "url": str(record.url), + "status_code": record.status_code, + "content_type": record.content_type, + "body": record.body, + "metadata_json": record.metadata, + "retrieved_at": record.retrieved_at, + }, + ) + context.storage.upsert( + session, + Document, + {"raw_document_id": raw.id}, + { + "corpus": context.settings.corpus_name, + "url": str(record.url), + "title": payload.get("title"), + "author": payload.get("author"), + "published_at": payload.get("date"), + "text": text, + "content_hash": content_hash(text), + "metadata_json": metadata, + "is_duplicate": False, + "canonical_document_id": None, + }, + ) + normalized += 1 + if normalized == 0: + raise ValueError("source handoff contained no records") + return {"normalized": normalized} + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + return run(context, input_jsonl=args.input_jsonl) + + +if __name__ == "__main__": + stage_main(1, run_from_args) diff --git a/src/02_clean_content.py b/src/02_clean_content.py new file mode 100644 index 0000000..cd3bb3f --- /dev/null +++ b/src/02_clean_content.py @@ -0,0 +1,103 @@ +from __future__ import annotations + +import argparse +import hashlib +import re + +from sqlalchemy import delete, select + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import Chunk, Document +from graphrag_platform.runtime import StageContext + +_SPACE_RE = re.compile(r"[ \t]+") +_PARAGRAPH_RE = re.compile(r"\n{3,}") + + +def content_hash(text: str) -> str: + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def clean_text(text: str) -> str: + lines = [_SPACE_RE.sub(" ", line).strip() for line in text.splitlines()] + return _PARAGRAPH_RE.sub("\n\n", "\n".join(lines)).strip() + + +def chunk_text(text: str, *, size: int, overlap: int) -> list[str]: + if overlap >= size: + raise ValueError("CHUNK_OVERLAP must be smaller than CHUNK_SIZE") + chunks: list[str] = [] + start = 0 + while start < len(text): + end = min(start + size, len(text)) + if end < len(text): + break_at = text.rfind(" ", start + size // 2, end) + if break_at > start: + end = break_at + chunk = text[start:end].strip() + if chunk: + chunks.append(chunk) + if end >= len(text): + break + start = end - overlap + return chunks + + +def run(context: StageContext) -> dict[str, int]: + context.initialize() + chunk_count = 0 + duplicate_count = 0 + with context.storage.session() as session: + documents = session.scalars( + select(Document) + .where(Document.corpus == context.settings.corpus_name) + .order_by(Document.id) + ).all() + if not documents: + raise ValueError("stage 2 requires normalized documents") + session.execute( + delete(Chunk).where(Chunk.corpus == context.settings.corpus_name) + ) + hashes: dict[str, int] = {} + for document in documents: + text = clean_text(document.text) + digest = content_hash(text) + document.text = text + document.content_hash = digest + if digest in hashes: + document.is_duplicate = True + document.canonical_document_id = hashes[digest] + duplicate_count += 1 + continue + document.is_duplicate = False + document.canonical_document_id = None + hashes[digest] = document.id + for ordinal, text_chunk in enumerate( + chunk_text( + text, + size=context.settings.chunk_size, + overlap=context.settings.chunk_overlap, + ) + ): + session.add( + Chunk( + document_id=document.id, + corpus=context.settings.corpus_name, + ordinal=ordinal, + text=text_chunk, + content_hash=content_hash(text_chunk), + ) + ) + chunk_count += 1 + return {"chunks": chunk_count, "duplicates": duplicate_count} + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + del args + return run(context) + + +if __name__ == "__main__": + stage_main(2, run_from_args) diff --git a/src/03_define_knowledge_model.py b/src/03_define_knowledge_model.py new file mode 100644 index 0000000..2b8c25e --- /dev/null +++ b/src/03_define_knowledge_model.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +from typing import Any + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import KnowledgeModel +from graphrag_platform.runtime import StageContext + + +def validate_model_definition(definition: dict[str, Any]) -> None: + for key in ("entity_types", "relationship_types"): + value = definition.get(key) + if not isinstance(value, list) or not value or not all( + isinstance(item, str) and item.strip() for item in value + ): + raise ValueError(f"{key} must be a non-empty array of strings") + for key in ("node_template", "edge_template"): + if not isinstance(definition.get(key), dict): + raise ValueError(f"{key} must be a JSON object") + + +def run( + context: StageContext, *, model_file: Path | None = None +) -> dict[str, int]: + context.initialize() + definition = context.settings.model_definition() + if model_file is not None: + loaded = json.loads(model_file.read_text(encoding="utf-8")) + if not isinstance(loaded, dict): + raise ValueError("knowledge model file must contain a JSON object") + definition.update(loaded) + validate_model_definition(definition) + with context.storage.session() as session: + context.storage.upsert( + session, + KnowledgeModel, + {"corpus": context.settings.corpus_name}, + {"definition_json": definition}, + ) + return { + "entity_types": len(definition["entity_types"]), + "relationship_types": len(definition["relationship_types"]), + } + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + return run(context, model_file=args.model_file) + + +if __name__ == "__main__": + stage_main(3, run_from_args) diff --git a/src/04_extract_knowledge.py b/src/04_extract_knowledge.py new file mode 100644 index 0000000..c98acb6 --- /dev/null +++ b/src/04_extract_knowledge.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import argparse +import json +from typing import Any + +from sqlalchemy import select + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import Chunk, Extraction, KnowledgeModel +from graphrag_platform.runtime import StageContext + + +def validate_extraction( + payload: dict[str, Any], definition: dict[str, Any] +) -> None: + entities = payload.get("entities") + relationships = payload.get("relationships") + if not isinstance(entities, list) or not isinstance(relationships, list): + raise ValueError("extraction requires entities and relationships arrays") + entity_types = set(definition["entity_types"]) + relationship_types = set(definition["relationship_types"]) + for entity in entities: + if not isinstance(entity, dict): + raise ValueError("each entity must be an object") + if not str(entity.get("canonical_name", "")).strip(): + raise ValueError("entity canonical_name must not be empty") + if entity.get("entity_type") not in entity_types: + raise ValueError(f"unsupported entity type: {entity.get('entity_type')}") + if not isinstance(entity.get("aliases", []), list): + raise ValueError("entity aliases must be an array") + for relationship in relationships: + if not isinstance(relationship, dict): + raise ValueError("each relationship must be an object") + if not str(relationship.get("source", "")).strip() or not str( + relationship.get("target", "") + ).strip(): + raise ValueError("relationship source and target must not be empty") + if relationship.get("relationship_type") not in relationship_types: + raise ValueError( + "unsupported relationship type: " + f"{relationship.get('relationship_type')}" + ) + + +def run(context: StageContext) -> dict[str, int]: + context.initialize() + with context.storage.session() as session: + model = session.get(KnowledgeModel, context.settings.corpus_name) + chunks = session.scalars( + select(Chunk).where(Chunk.corpus == context.settings.corpus_name) + ).all() + if model is None or not chunks: + raise ValueError("stage 4 requires a knowledge model and cleaned chunks") + definition = dict(model.definition_json) + chunk_inputs = [(chunk.id, chunk.text) for chunk in chunks] + + system = ( + "You extract a source-grounded knowledge graph. Return one JSON object " + 'with arrays named "entities" and "relationships". Never infer unsupported facts.' + ) + extracted_count = 0 + for chunk_id, chunk_text in chunk_inputs: + prompt = json.dumps( + { + "knowledge_model": definition, + "source_chunk_id": chunk_id, + "text": chunk_text, + }, + ensure_ascii=False, + ) + payload = context.llm.complete_json(system, prompt) + validate_extraction(payload, definition) + with context.storage.session() as session: + context.storage.upsert( + session, + Extraction, + {"chunk_id": chunk_id}, + { + "corpus": context.settings.corpus_name, + "payload_json": payload, + "model_name": context.llm.chat_model, + }, + ) + extracted_count += 1 + return {"extracted_chunks": extracted_count} + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + del args + return run(context) + + +if __name__ == "__main__": + stage_main(4, run_from_args) diff --git a/src/05_reconcile_knowledge.py b/src/05_reconcile_knowledge.py new file mode 100644 index 0000000..2dba4c9 --- /dev/null +++ b/src/05_reconcile_knowledge.py @@ -0,0 +1,137 @@ +from __future__ import annotations + +import argparse +import re +from typing import Any + +from sqlalchemy import delete, select + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import ( + Extraction, + ResolvedEntity, + ResolvedRelationship, +) +from graphrag_platform.runtime import StageContext + +_KEY_RE = re.compile(r"[^\w]+", re.UNICODE) + + +def canonical_key(name: str) -> str: + return _KEY_RE.sub(" ", name.casefold()).strip() + + +def merge_text(current: str, incoming: str) -> str: + current = current.strip() + incoming = incoming.strip() + if not incoming or incoming.casefold() in current.casefold(): + return current + if not current: + return incoming + return f"{current}\n{incoming}" + + +def run(context: StageContext) -> dict[str, int]: + context.initialize() + entity_count = 0 + relationship_count = 0 + with context.storage.session() as session: + extractions = session.scalars( + select(Extraction).where( + Extraction.corpus == context.settings.corpus_name + ) + ).all() + if not extractions: + raise ValueError("stage 5 requires extracted knowledge") + session.execute( + delete(ResolvedRelationship).where( + ResolvedRelationship.corpus == context.settings.corpus_name + ) + ) + session.execute( + delete(ResolvedEntity).where( + ResolvedEntity.corpus == context.settings.corpus_name + ) + ) + entity_rows: dict[str, dict[str, Any]] = {} + relation_rows: dict[tuple[str, str, str], dict[str, Any]] = {} + pending_relationships: list[tuple[int, dict[str, Any]]] = [] + for extraction in extractions: + payload = extraction.payload_json + for entity in payload["entities"]: + name = entity["canonical_name"].strip() + key = canonical_key(name) + row = entity_rows.get(key) + if row is None: + row = { + "canonical_name": name, + "entity_type": entity["entity_type"], + "description": "", + "aliases": set(), + "chunk_ids": set(), + } + entity_rows[key] = row + elif row["entity_type"] != entity["entity_type"]: + raise ValueError( + f"conflicting entity types for {name}: " + f"{row['entity_type']} and {entity['entity_type']}" + ) + row["description"] = merge_text( + row["description"], entity.get("description", "") + ) + row["aliases"].update(entity.get("aliases", [])) + row["aliases"].add(name) + row["chunk_ids"].add(extraction.chunk_id) + for relationship in payload["relationships"]: + pending_relationships.append((extraction.chunk_id, relationship)) + for chunk_id, relationship in pending_relationships: + source_key = canonical_key(relationship["source"]) + target_key = canonical_key(relationship["target"]) + if source_key not in entity_rows or target_key not in entity_rows: + continue + relation_type = relationship["relationship_type"] + identity = (source_key, target_key, relation_type) + row = relation_rows.setdefault( + identity, {"description": "", "chunk_ids": set()} + ) + row["description"] = merge_text( + row["description"], relationship.get("description", "") + ) + row["chunk_ids"].add(chunk_id) + for key, row in entity_rows.items(): + session.add( + ResolvedEntity( + corpus=context.settings.corpus_name, + canonical_key=key, + canonical_name=row["canonical_name"], + entity_type=row["entity_type"], + description=row["description"], + aliases_json=sorted(row["aliases"], key=str.casefold), + source_chunk_ids_json=sorted(row["chunk_ids"]), + ) + ) + entity_count += 1 + for (source_key, target_key, relation_type), row in relation_rows.items(): + session.add( + ResolvedRelationship( + corpus=context.settings.corpus_name, + source_key=source_key, + target_key=target_key, + relationship_type=relation_type, + description=row["description"], + source_chunk_ids_json=sorted(row["chunk_ids"]), + ) + ) + relationship_count += 1 + return {"entities": entity_count, "relationships": relationship_count} + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + del args + return run(context) + + +if __name__ == "__main__": + stage_main(5, run_from_args) diff --git a/src/06_build_knowledge_graph.py b/src/06_build_knowledge_graph.py new file mode 100644 index 0000000..a760cbc --- /dev/null +++ b/src/06_build_knowledge_graph.py @@ -0,0 +1,80 @@ +from __future__ import annotations + +import argparse + +from sqlalchemy import delete, select + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import ( + Edge, + Node, + ResolvedEntity, + ResolvedRelationship, +) +from graphrag_platform.runtime import StageContext + + +def run(context: StageContext) -> dict[str, int]: + context.initialize() + with context.storage.session() as session: + entities = session.scalars( + select(ResolvedEntity).where( + ResolvedEntity.corpus == context.settings.corpus_name + ) + ).all() + relationships = session.scalars( + select(ResolvedRelationship).where( + ResolvedRelationship.corpus == context.settings.corpus_name + ) + ).all() + if not entities: + raise ValueError("stage 6 requires reconciled entities") + session.execute( + delete(Edge).where(Edge.corpus == context.settings.corpus_name) + ) + session.execute( + delete(Node).where(Node.corpus == context.settings.corpus_name) + ) + node_ids: dict[str, int] = {} + for entity in entities: + node = Node( + corpus=entity.corpus, + canonical_key=entity.canonical_key, + canonical_name=entity.canonical_name, + entity_type=entity.entity_type, + description=entity.description, + aliases_json=entity.aliases_json, + source_chunk_ids_json=entity.source_chunk_ids_json, + ) + session.add(node) + session.flush() + node_ids[entity.canonical_key] = node.id + edge_count = 0 + for relationship in relationships: + source_id = node_ids.get(relationship.source_key) + target_id = node_ids.get(relationship.target_key) + if source_id is None or target_id is None: + continue + session.add( + Edge( + corpus=relationship.corpus, + source_node_id=source_id, + target_node_id=target_id, + relationship_type=relationship.relationship_type, + description=relationship.description, + source_chunk_ids_json=relationship.source_chunk_ids_json, + ) + ) + edge_count += 1 + return {"nodes": len(node_ids), "edges": edge_count} + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + del args + return run(context) + + +if __name__ == "__main__": + stage_main(6, run_from_args) diff --git a/src/07_build_communities.py b/src/07_build_communities.py new file mode 100644 index 0000000..89ff15f --- /dev/null +++ b/src/07_build_communities.py @@ -0,0 +1,97 @@ +from __future__ import annotations + +import argparse +import json + +import networkx as nx +from sqlalchemy import delete, select + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import Community, Edge, Node +from graphrag_platform.runtime import StageContext + + +def run(context: StageContext) -> dict[str, int]: + context.initialize() + with context.storage.session() as session: + nodes = session.scalars( + select(Node).where(Node.corpus == context.settings.corpus_name) + ).all() + edges = session.scalars( + select(Edge).where(Edge.corpus == context.settings.corpus_name) + ).all() + if not nodes: + raise ValueError("stage 7 requires graph nodes") + graph = nx.Graph() + graph.add_nodes_from(node.id for node in nodes) + graph.add_edges_from( + (edge.source_node_id, edge.target_node_id) for edge in edges + ) + if graph.number_of_edges(): + groups = list( + nx.community.greedy_modularity_communities( + graph, resolution=context.settings.community_resolution + ) + ) + else: + groups = [{node.id} for node in nodes] + by_id = {node.id: node for node in nodes} + community_inputs = [ + ( + index, + sorted(group), + [ + { + "name": by_id[node_id].canonical_name, + "type": by_id[node_id].entity_type, + "description": by_id[node_id].description, + } + for node_id in sorted(group) + ], + ) + for index, group in enumerate(groups, start=1) + ] + + summaries: list[tuple[int, list[int], str, str]] = [] + for index, node_ids, evidence in community_inputs: + summary = context.llm.complete_json( + ( + "Summarize a knowledge-graph community using only the supplied " + 'evidence. Return JSON with non-empty "title" and "summary" strings.' + ), + json.dumps(evidence, ensure_ascii=False), + ) + title = str(summary.get("title", "")).strip() + description = str(summary.get("summary", "")).strip() + if not title or not description: + raise ValueError("community summary requires title and summary") + summaries.append((index, node_ids, title, description)) + + with context.storage.session() as session: + session.execute( + delete(Community).where( + Community.corpus == context.settings.corpus_name + ) + ) + for index, node_ids, title, description in summaries: + session.add( + Community( + corpus=context.settings.corpus_name, + community_key=f"community-{index}", + title=title, + summary=description, + node_ids_json=node_ids, + ) + ) + return {"communities": len(groups)} + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + del args + return run(context) + + +if __name__ == "__main__": + stage_main(7, run_from_args) diff --git a/src/08_build_retrieval_index.py b/src/08_build_retrieval_index.py new file mode 100644 index 0000000..bdf5edb --- /dev/null +++ b/src/08_build_retrieval_index.py @@ -0,0 +1,164 @@ +from __future__ import annotations + +import argparse +from typing import Any + +from sqlalchemy import delete, select + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import ( + Chunk, + Community, + Document, + Edge, + Node, + SearchItem, +) +from graphrag_platform.runtime import StageContext + + +def run(context: StageContext) -> dict[str, int]: + context.initialize() + with context.storage.session() as session: + chunks = session.scalars( + select(Chunk).where(Chunk.corpus == context.settings.corpus_name) + ).all() + nodes = session.scalars( + select(Node).where(Node.corpus == context.settings.corpus_name) + ).all() + edges = session.scalars( + select(Edge).where(Edge.corpus == context.settings.corpus_name) + ).all() + communities = session.scalars( + select(Community).where( + Community.corpus == context.settings.corpus_name + ) + ).all() + if not nodes or not chunks: + raise ValueError("stage 8 requires graph nodes and source chunks") + documents = { + document.id: document + for document in session.scalars( + select(Document).where( + Document.corpus == context.settings.corpus_name + ) + ).all() + } + node_names = {node.id: node.canonical_name for node in nodes} + node_source_chunks = { + node.id: set(node.source_chunk_ids_json) for node in nodes + } + candidates: list[dict[str, Any]] = [] + for chunk in chunks: + document = documents[chunk.document_id] + candidates.append( + { + "item_type": "chunk", + "item_key": str(chunk.id), + "text": chunk.text, + "source_url": document.url, + "metadata": { + "document_id": document.id, + "title": document.title, + "chunk_ordinal": chunk.ordinal, + }, + } + ) + for node in nodes: + candidates.append( + { + "item_type": "node", + "item_key": str(node.id), + "text": ( + f"{node.canonical_name} ({node.entity_type}): " + f"{node.description}" + ), + "source_url": None, + "metadata": {"source_chunk_ids": node.source_chunk_ids_json}, + } + ) + for edge in edges: + candidates.append( + { + "item_type": "edge", + "item_key": str(edge.id), + "text": ( + f"{node_names[edge.source_node_id]} " + f"{edge.relationship_type} " + f"{node_names[edge.target_node_id]}: {edge.description}" + ), + "source_url": None, + "metadata": { + "source_node_id": edge.source_node_id, + "target_node_id": edge.target_node_id, + "relationship_type": edge.relationship_type, + "source_chunk_ids": edge.source_chunk_ids_json, + }, + } + ) + for community in communities: + community_chunk_ids: set[int] = set() + for node_id in community.node_ids_json: + community_chunk_ids.update(node_source_chunks.get(node_id, set())) + candidates.append( + { + "item_type": "community", + "item_key": str(community.id), + "text": f"{community.title}: {community.summary}", + "source_url": None, + "metadata": { + "node_ids": community.node_ids_json, + "source_chunk_ids": sorted(community_chunk_ids), + }, + } + ) + embeddings: list[list[float]] = [] + for offset in range(0, len(candidates), 64): + embeddings.extend( + context.llm.embed( + [row["text"] for row in candidates[offset : offset + 64]] + ) + ) + if len(embeddings) != len(candidates): + raise ValueError("embedding provider returned an unexpected row count") + dimensions = context.settings.llm_embedding_dimensions + invalid_dimensions = [ + len(embedding) for embedding in embeddings if len(embedding) != dimensions + ] + if invalid_dimensions: + raise ValueError( + "embedding provider returned vectors with unexpected dimensions: " + f"expected {dimensions}, received {invalid_dimensions[0]}" + ) + with context.storage.session() as session: + session.execute( + delete(SearchItem).where( + SearchItem.corpus == context.settings.corpus_name + ) + ) + for candidate, embedding in zip(candidates, embeddings, strict=True): + session.add( + SearchItem( + corpus=context.settings.corpus_name, + item_type=candidate["item_type"], + item_key=candidate["item_key"], + text=candidate["text"], + source_url=candidate["source_url"], + metadata_json=candidate["metadata"], + embedding=embedding, + embedding_model=context.llm.embedding_model, + ) + ) + context.storage.ensure_vector_index(context.settings.llm_embedding_dimensions) + return {"indexed_items": len(candidates)} + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, int]: + del args + return run(context) + + +if __name__ == "__main__": + stage_main(8, run_from_args) diff --git a/src/09_query_graphrag.py b/src/09_query_graphrag.py new file mode 100644 index 0000000..84c8661 --- /dev/null +++ b/src/09_query_graphrag.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +import argparse +import math +from typing import Any + +from pgvector.sqlalchemy import Vector +from sqlalchemy import func, or_, select, text +from sqlalchemy.orm import Session + +from graphrag_platform.cli import stage_main +from graphrag_platform.models import Chunk, Document, Edge, Node, SearchItem +from graphrag_platform.runtime import StageContext + + +def cosine_similarity(left: list[float], right: list[float]) -> float: + if len(left) != len(right) or not left: + return 0.0 + denominator = math.sqrt(sum(x * x for x in left)) * math.sqrt( + sum(x * x for x in right) + ) + if denominator == 0: + return 0.0 + return sum(x * y for x, y in zip(left, right, strict=True)) / denominator + + +def sources_for_chunks( + session: Session, chunk_ids: set[int] +) -> list[dict[str, Any]]: + if not chunk_ids: + return [] + rows = session.execute( + select(Chunk, Document) + .join(Document, Chunk.document_id == Document.id) + .where(Chunk.id.in_(chunk_ids)) + .order_by(Document.url, Chunk.ordinal) + ).all() + return [ + { + "citation": f"S{index}", + "url": document.url, + "title": document.title, + "chunk_id": chunk.id, + "text": chunk.text, + } + for index, (chunk, document) in enumerate(rows, start=1) + ] + + +def rank_vector_context( + context: StageContext, + session: Session, + query_embedding: list[float], + limit: int, +) -> list[tuple[float, SearchItem]]: + if context.storage.supports_vector_search: + dimensions = context.settings.llm_embedding_dimensions + session.execute( + text( + "SET LOCAL hnsw.ef_search = " + f"{context.settings.retrieval_hnsw_ef_search}" + ) + ) + session.execute(text("SET LOCAL hnsw.iterative_scan = strict_order")) + distance = ( + SearchItem.embedding.cast(Vector(dimensions)) + .cosine_distance(query_embedding) + .label("distance") + ) + rows = session.execute( + select(SearchItem, distance) + .where( + SearchItem.corpus == context.settings.corpus_name, + SearchItem.embedding_model == context.llm.embedding_model, + func.vector_dims(SearchItem.embedding) == dimensions, + ) + .order_by(distance) + .limit(limit) + ).all() + return [(1.0 - float(distance_value), item) for item, distance_value in rows] + + search_items = session.scalars( + select(SearchItem).where( + SearchItem.corpus == context.settings.corpus_name, + SearchItem.embedding_model == context.llm.embedding_model, + ) + ).all() + return sorted( + ( + (cosine_similarity(query_embedding, list(item.embedding)), item) + for item in search_items + ), + key=lambda pair: pair[0], + reverse=True, + )[:limit] + + +def expand_graph( + context: StageContext, + session: Session, + ranked: list[tuple[float, SearchItem]], +) -> tuple[list[dict[str, Any]], set[int]]: + seed_node_ids: set[int] = set() + source_chunk_ids: set[int] = set() + for _, item in ranked: + source_chunk_ids.update(item.metadata_json.get("source_chunk_ids", [])) + if item.item_type == "node": + seed_node_ids.add(int(item.item_key)) + elif item.item_type == "edge": + seed_node_ids.update( + { + int(item.metadata_json["source_node_id"]), + int(item.metadata_json["target_node_id"]), + } + ) + elif item.item_type == "community": + seed_node_ids.update(item.metadata_json.get("node_ids", [])) + + if not seed_node_ids or context.settings.retrieval_graph_hops == 0: + return [], source_chunk_ids + + visited_nodes = set(seed_node_ids) + visited_edges: dict[int, Edge] = {} + frontier = set(seed_node_ids) + for _ in range(context.settings.retrieval_graph_hops): + if not frontier or len(visited_edges) >= context.settings.retrieval_graph_limit: + break + remaining = context.settings.retrieval_graph_limit - len(visited_edges) + edges = session.scalars( + select(Edge) + .where( + Edge.corpus == context.settings.corpus_name, + or_( + Edge.source_node_id.in_(frontier), + Edge.target_node_id.in_(frontier), + ), + ) + .order_by(Edge.weight.desc(), Edge.id) + .limit(remaining) + ).all() + next_frontier: set[int] = set() + for edge in edges: + visited_edges[edge.id] = edge + source_chunk_ids.update(edge.source_chunk_ids_json) + next_frontier.update({edge.source_node_id, edge.target_node_id}) + next_frontier -= visited_nodes + visited_nodes.update(next_frontier) + frontier = next_frontier + + nodes = { + node.id: node + for node in session.scalars( + select(Node).where( + Node.corpus == context.settings.corpus_name, + Node.id.in_(visited_nodes), + ) + ).all() + } + for node in nodes.values(): + source_chunk_ids.update(node.source_chunk_ids_json) + graph_context = [ + { + "edge_id": edge.id, + "source_node_id": edge.source_node_id, + "source": nodes[edge.source_node_id].canonical_name, + "relationship": edge.relationship_type, + "target_node_id": edge.target_node_id, + "target": nodes[edge.target_node_id].canonical_name, + "description": edge.description, + "weight": edge.weight, + "source_chunk_ids": edge.source_chunk_ids_json, + } + for edge in visited_edges.values() + if edge.source_node_id in nodes and edge.target_node_id in nodes + ] + return graph_context, source_chunk_ids + + +def run( + context: StageContext, + *, + question: str, + top_k: int | None = None, +) -> dict[str, Any]: + context.initialize() + if not question.strip(): + raise ValueError("question must not be empty") + query_embedding = context.llm.embed([question])[0] + expected_dimensions = context.settings.llm_embedding_dimensions + if len(query_embedding) != expected_dimensions: + raise ValueError( + "embedding provider returned a query vector with unexpected dimensions: " + f"expected {expected_dimensions}, received {len(query_embedding)}" + ) + with context.storage.session() as session: + limit = top_k or context.settings.retrieval_top_k + ranked = rank_vector_context( + context, + session, + query_embedding, + limit, + ) + if not ranked: + raise ValueError("stage 9 requires a retrieval index") + graph_context, chunk_ids = expand_graph(context, session, ranked) + for _, item in ranked: + if item.item_type == "chunk": + chunk_ids.add(int(item.item_key)) + chunk_ids.update(item.metadata_json.get("source_chunk_ids", [])) + sources = sources_for_chunks(session, chunk_ids) + return { + "question": question, + "context": [ + { + "score": round(score, 6), + "type": item.item_type, + "key": item.item_key, + "text": item.text, + "source_url": item.source_url, + "metadata": item.metadata_json, + } + for score, item in ranked + ], + "graph_context": graph_context, + "sources": sources, + } + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, Any]: + if not args.question: + raise ValueError("--question is required for stage 9") + return run(context, question=args.question, top_k=args.top_k) + + +if __name__ == "__main__": + stage_main(9, run_from_args) diff --git a/src/10_generate_answer.py b/src/10_generate_answer.py new file mode 100644 index 0000000..dd6c26c --- /dev/null +++ b/src/10_generate_answer.py @@ -0,0 +1,47 @@ +from __future__ import annotations + +import argparse +import json +from typing import Any + +from graphrag_platform.cli import stage_main +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import load_stage + + +def run( + context: StageContext, + *, + question: str, + top_k: int | None = None, +) -> dict[str, Any]: + retrieval = load_stage(9).run( + context, + question=question, + top_k=top_k, + ) + answer = context.llm.complete_text( + ( + "Answer only from the supplied GraphRAG context. Cite supporting sources " + "using [S1], [S2], and so on. If the context does not support an answer, " + "state that clearly. Do not invent citations." + ), + json.dumps(retrieval, ensure_ascii=False), + ) + return { + "question": question, + "answer": answer, + "sources": retrieval["sources"], + } + + +def run_from_args( + context: StageContext, args: argparse.Namespace +) -> dict[str, Any]: + if not args.question: + raise ValueError("--question is required for stage 10") + return run(context, question=args.question, top_k=args.top_k) + + +if __name__ == "__main__": + stage_main(10, run_from_args) diff --git a/src/graphrag_platform/__init__.py b/src/graphrag_platform/__init__.py new file mode 100644 index 0000000..b5ea6eb --- /dev/null +++ b/src/graphrag_platform/__init__.py @@ -0,0 +1,6 @@ +"""KnowledgeGraph GraphRAG platform.""" + +from graphrag_platform.config import Settings +from graphrag_platform.pipeline import KnowledgeGraphPipeline + +__all__ = ["KnowledgeGraphPipeline", "Settings"] \ No newline at end of file diff --git a/src/graphrag_platform/api.py b/src/graphrag_platform/api.py new file mode 100644 index 0000000..e0de338 --- /dev/null +++ b/src/graphrag_platform/api.py @@ -0,0 +1,247 @@ +from __future__ import annotations + +from typing import Annotated, Any, Callable + +import jwt +from fastapi import Depends, FastAPI, HTTPException, Request, status +from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer +from pydantic import BaseModel, Field +from sqlalchemy import select, text + +from graphrag_platform.auth import Permission, Principal, decode_internal_token +from graphrag_platform.config import Settings +from graphrag_platform.llm import LLMClient +from graphrag_platform.models import PipelineRun, StageRun +from graphrag_platform.observability import configure_logging +from graphrag_platform.orchestrator import requeue_failed_run, submit_run +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import load_stage +from graphrag_platform.storage import Storage + +bearer = HTTPBearer(auto_error=False) + + +class JobRequest(BaseModel): + corpus: str = Field(min_length=1, max_length=200) + from_stage: int = Field(default=1, ge=1, le=8) + through_stage: int = Field(default=8, ge=1, le=8) + input_jsonl: str | None = None + model_file: str | None = None + reset: bool = False + + +class QueryRequest(BaseModel): + corpus: str = Field(min_length=1, max_length=200) + question: str = Field(min_length=1) + top_k: int | None = Field(default=None, ge=1) + + +def _principal( + request: Request, + credentials: Annotated[ + HTTPAuthorizationCredentials | None, Depends(bearer) + ], +) -> Principal: + if credentials is None: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Bearer service token is required", + ) + try: + return decode_internal_token(credentials.credentials, request.app.state.settings) + except jwt.PyJWTError as error: + raise HTTPException( + status_code=status.HTTP_401_UNAUTHORIZED, + detail="Invalid service token", + ) from error + + +def require_permissions( + required: Permission, +) -> Callable[[Principal], Principal]: + def dependency(principal: Annotated[Principal, Depends(_principal)]) -> Principal: + if not principal.has(required): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail=f"KnowledgeGraph permission mask {int(required)} is required", + ) + return principal + + return dependency + + +def create_app( + settings: Settings | None = None, + *, + llm: LLMClient | None = None, +) -> FastAPI: + active_settings = settings or Settings() + configure_logging(active_settings.log_level) + app = FastAPI( + title="KnowledgeGraph Internal API", + version="1.0.0", + docs_url=None, + redoc_url=None, + openapi_url="/internal/knowledge-graph/openapi.json", + ) + app.state.settings = active_settings + app.state.storage = Storage(active_settings.database_url) + app.state.llm = llm + + def context(corpus: str) -> StageContext: + corpus_settings = active_settings.model_copy(update={"corpus_name": corpus}) + return StageContext.create( + corpus_settings, + storage=app.state.storage, + llm=app.state.llm, + ) + + @app.get("/internal/knowledge-graph/health") + def health() -> dict[str, str]: + return {"status": "healthy"} + + @app.get("/internal/knowledge-graph/ready") + def ready() -> dict[str, str]: + try: + with app.state.storage.engine.connect() as connection: + connection.execute(text("SELECT 1")) + if connection.dialect.name == "postgresql": + extension = connection.scalar( + text( + "SELECT extname FROM pg_extension " + "WHERE extname = 'vector'" + ) + ) + if extension != "vector": + raise RuntimeError("PgVector extension is not installed") + except Exception as error: + raise HTTPException( + status_code=status.HTTP_503_SERVICE_UNAVAILABLE, + detail="Service dependencies are not ready", + ) from error + return {"status": "ready"} + + @app.post( + "/internal/knowledge-graph/jobs", + status_code=status.HTTP_202_ACCEPTED, + ) + def create_job( + body: JobRequest, + principal: Annotated[ + Principal, Depends(require_permissions(Permission.EXECUTE)) + ], + ) -> dict[str, str]: + if body.from_stage == 1 and not principal.has(Permission.ADD): + raise HTTPException( + status_code=status.HTTP_403_FORBIDDEN, + detail="KnowledgeGraph Add permission is required for stage 1", + ) + if body.from_stage > body.through_stage: + raise HTTPException( + status_code=status.HTTP_422_UNPROCESSABLE_ENTITY, + detail="from_stage must not exceed through_stage", + ) + run_context = context(body.corpus) + run_context.initialize() + run_id = submit_run( + run_context, + from_stage=body.from_stage, + through_stage=body.through_stage, + payload={ + "input_jsonl": body.input_jsonl, + "model_file": body.model_file, + "reset": body.reset, + }, + requested_by=principal.subject, + correlation_id=principal.correlation_id, + permission_mask=principal.permission_mask, + ) + return {"job_id": run_id, "status": "queued"} + + @app.get("/internal/knowledge-graph/jobs/{job_id}") + def get_job( + job_id: str, + principal: Annotated[ + Principal, Depends(require_permissions(Permission.READ)) + ], + ) -> dict[str, Any]: + del principal + with app.state.storage.session() as session: + run = session.get(PipelineRun, job_id) + if run is None: + raise HTTPException(status_code=404, detail="Job not found") + stages = session.scalars( + select(StageRun) + .where(StageRun.pipeline_run_id == job_id) + .order_by(StageRun.stage_number, StageRun.attempt) + ).all() + return { + "job_id": run.id, + "corpus": run.corpus, + "status": run.status, + "attempt": run.attempt, + "from_stage": run.from_stage, + "through_stage": run.through_stage, + "error": run.error_message, + "output": run.output_json, + "stages": [ + { + "stage": stage_run.stage_number, + "name": stage_run.stage_name, + "status": stage_run.status, + "attempt": stage_run.attempt, + "output": stage_run.output_json, + "error": stage_run.error_message, + } + for stage_run in stages + ], + } + + @app.post( + "/internal/knowledge-graph/jobs/{job_id}/retry", + status_code=status.HTTP_202_ACCEPTED, + ) + def retry_job( + job_id: str, + principal: Annotated[ + Principal, Depends(require_permissions(Permission.EXECUTE)) + ], + ) -> dict[str, str]: + del principal + requeue_failed_run(context(active_settings.corpus_name), job_id) + return {"job_id": job_id, "status": "queued"} + + @app.post("/internal/knowledge-graph/query") + def query( + body: QueryRequest, + principal: Annotated[ + Principal, + Depends(require_permissions(Permission.READ | Permission.EXECUTE)), + ], + ) -> dict[str, Any]: + del principal + return load_stage(9).run( + context(body.corpus), + question=body.question, + top_k=body.top_k, + ) + + @app.post("/internal/knowledge-graph/answer") + def answer( + body: QueryRequest, + principal: Annotated[ + Principal, + Depends(require_permissions(Permission.READ | Permission.EXECUTE)), + ], + ) -> dict[str, Any]: + del principal + return load_stage(10).run( + context(body.corpus), + question=body.question, + top_k=body.top_k, + ) + + return app + + +app = create_app() diff --git a/src/graphrag_platform/auth.py b/src/graphrag_platform/auth.py new file mode 100644 index 0000000..db58a83 --- /dev/null +++ b/src/graphrag_platform/auth.py @@ -0,0 +1,56 @@ +from __future__ import annotations + +from dataclasses import dataclass +from enum import IntFlag +from typing import Any + +import jwt + +from graphrag_platform.config import Settings + + +class Permission(IntFlag): + READ = 1 + ADD = 2 + EDIT = 4 + DELETE = 8 + EXECUTE = 16 + + +@dataclass(frozen=True) +class Principal: + subject: str + permission_mask: int + correlation_id: str + claims: dict[str, Any] + + def has(self, required: Permission) -> bool: + return (self.permission_mask & int(required)) == int(required) + + +def decode_internal_token(token: str, settings: Settings) -> Principal: + claims = jwt.decode( + token, + settings.internal_jwt_secret, + algorithms=["HS256"], + issuer=settings.internal_jwt_issuer, + audience=settings.internal_jwt_audience, + options={"require": ["exp", "iat", "iss", "aud", "sub", "features"]}, + ) + features = claims.get("features") + if not isinstance(features, dict): + raise jwt.InvalidTokenError("features claim must be an object") + raw_mask = features.get(settings.internal_feature_name) + if not isinstance(raw_mask, int): + raise jwt.InvalidTokenError( + f"missing integer feature claim: {settings.internal_feature_name}" + ) + correlation_id = str(claims.get("correlation_id") or claims.get("jti") or "") + if not correlation_id: + raise jwt.InvalidTokenError("correlation_id or jti is required") + return Principal( + subject=str(claims["sub"]), + permission_mask=raw_mask, + correlation_id=correlation_id, + claims=claims, + ) diff --git a/src/graphrag_platform/cli.py b/src/graphrag_platform/cli.py new file mode 100644 index 0000000..331e9ae --- /dev/null +++ b/src/graphrag_platform/cli.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import argparse +import json +import sys +from uuid import uuid4 +from collections.abc import Callable +from pathlib import Path +from typing import Any + +from graphrag_platform.config import Settings +from graphrag_platform.observability import configure_logging +from graphrag_platform.orchestrator import execute_run, submit_run +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import STAGE_MODULES, load_stage + +StageRunner = Callable[[StageContext, argparse.Namespace], dict[str, Any]] + + +def create_parser(fixed_stage: int | None) -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Run a KnowledgeGraph GraphRAG pipeline stage." + ) + if fixed_stage is None: + parser.add_argument("stage", type=int, choices=range(1, 11)) + parser.add_argument("--env-file", type=Path, default=Path(".env")) + parser.add_argument("--database-url", help="Override DATABASE_URL.") + parser.add_argument("--corpus", help="Override CORPUS_NAME.") + parser.add_argument("--input-jsonl", type=Path) + parser.add_argument("--model-file", type=Path) + parser.add_argument("--question") + parser.add_argument("--top-k", type=int) + parser.add_argument("--reset", action="store_true") + return parser + + +def create_run_parser() -> argparse.ArgumentParser: + parser = create_parser(1) + parser.description = "Run a durable KnowledgeGraph build pipeline." + parser.add_argument("--from-stage", type=int, choices=range(1, 9), default=1) + parser.add_argument("--through-stage", type=int, choices=range(1, 9), default=8) + parser.add_argument("--requested-by", default="local-cli") + parser.add_argument("--correlation-id", default=None) + return parser + + +def create_context( + args: argparse.Namespace, stage: int, *, apply_reset: bool = True +) -> StageContext: + overrides: dict[str, Any] = {} + if args.database_url: + overrides["database_url"] = args.database_url + if args.corpus: + overrides["corpus_name"] = args.corpus + settings = Settings(_env_file=args.env_file, **overrides) + context = StageContext.create(settings) + context.initialize() + if args.reset and apply_reset: + context.storage.clear_from_stage(settings.corpus_name, stage) + return context + + +def run(stage: int, args: argparse.Namespace) -> dict[str, Any]: + context = create_context(args, stage) + module = load_stage(stage) + return module.run_from_args(context, args) + + +def stage_main(stage: int, runner: StageRunner) -> None: + args = create_parser(stage).parse_args() + context = create_context(args, stage) + configure_logging(context.settings.log_level) + result = runner(context, args) + print(json.dumps(result, indent=2, ensure_ascii=False)) + + +def main_all() -> None: + if len(sys.argv) > 1 and sys.argv[1] == "run": + args = create_run_parser().parse_args(sys.argv[2:]) + context = create_context(args, args.from_stage, apply_reset=False) + configure_logging(context.settings.log_level) + run_id = submit_run( + context, + from_stage=args.from_stage, + through_stage=args.through_stage, + payload={ + "input_jsonl": str(args.input_jsonl) if args.input_jsonl else None, + "model_file": str(args.model_file) if args.model_file else None, + "reset": args.reset, + }, + requested_by=args.requested_by, + correlation_id=args.correlation_id or str(uuid4()), + permission_mask=31, + ) + outputs = execute_run(context, run_id) + print(json.dumps({"run_id": run_id, "outputs": outputs}, indent=2)) + return + args = create_parser(None).parse_args() + configure_logging(Settings(_env_file=args.env_file).log_level) + result = run(args.stage, args) + print( + json.dumps( + { + "stage": args.stage, + "name": STAGE_MODULES[args.stage].removeprefix(f"{args.stage:02d}_"), + **result, + }, + indent=2, + ensure_ascii=False, + ) + ) diff --git a/src/graphrag_platform/config.py b/src/graphrag_platform/config.py new file mode 100644 index 0000000..2930fe4 --- /dev/null +++ b/src/graphrag_platform/config.py @@ -0,0 +1,126 @@ +from __future__ import annotations + +import json +from pathlib import Path +from typing import Annotated, Any + +from pydantic import Field, field_validator +from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict + + +class Settings(BaseSettings): + """Runtime configuration loaded from environment variables or a .env file.""" + + model_config = SettingsConfigDict( + env_file=".env", + env_file_encoding="utf-8", + extra="ignore", + case_sensitive=False, + ) + + database_url: str = ( + "postgresql+psycopg://knowledgegraph_app:change-me" + "@localhost:5432/knowledgegraph" + ) + source_database_url: str | None = None + source_table_name: str = "external_documents" + analytics_database_url: str | None = None + auto_create_schema: bool = False + + corpus_name: str = "default-corpus" + + entity_types: Annotated[list[str], NoDecode] = Field( + default_factory=lambda: [ + "Person", + "Organization", + "Product", + "Concept", + "Event", + "Location", + ] + ) + relationship_types: Annotated[list[str], NoDecode] = Field( + default_factory=lambda: [ + "AFFILIATED_WITH", + "CREATED", + "DEPENDS_ON", + "MENTIONS", + "PART_OF", + "RELATED_TO", + ] + ) + node_template: dict[str, Any] = Field( + default_factory=lambda: { + "canonical_name": "string", + "entity_type": "one configured entity type", + "description": "source-grounded string", + "aliases": ["string"], + } + ) + edge_template: dict[str, Any] = Field( + default_factory=lambda: { + "source": "entity name", + "target": "entity name", + "relationship_type": "one configured relationship type", + "description": "source-grounded string", + } + ) + extraction_instructions: str = ( + "Extract only entities and relationships explicitly supported by the supplied text." + ) + + llm_base_url: str = "https://api.openai.com/v1" + llm_api_key: str | None = None + llm_chat_model: str = "gpt-4.1-mini" + llm_embedding_model: str = "text-embedding-3-small" + llm_embedding_dimensions: int = Field(default=1536, ge=1) + llm_timeout_seconds: float = Field(default=60, gt=0) + llm_max_retries: int = Field(default=4, ge=0, le=10) + llm_retry_initial_seconds: float = Field(default=1, gt=0) + llm_retry_max_seconds: float = Field(default=30, gt=0) + + chunk_size: int = Field(default=1200, ge=200) + chunk_overlap: int = Field(default=150, ge=0) + community_resolution: float = Field(default=1.0, gt=0) + retrieval_top_k: int = Field(default=8, ge=1) + retrieval_graph_hops: int = Field(default=2, ge=0, le=5) + retrieval_graph_limit: int = Field(default=50, ge=1) + retrieval_hnsw_ef_search: int = Field(default=100, ge=1) + log_level: str = "INFO" + internal_jwt_secret: str = "replace-in-dev-test" + internal_jwt_issuer: str = "knowledgegraph-bff" + internal_jwt_audience: str = "knowledgegraph-service" + internal_feature_name: str = "KnowledgeGraph" + worker_poll_seconds: float = Field(default=2, gt=0) + worker_stale_run_seconds: int = Field(default=3600, ge=60) + artifact_dir: Path = Path("artifacts") + + @field_validator( + "entity_types", + "relationship_types", + mode="before", + ) + @classmethod + def parse_comma_separated_list(cls, value: object) -> object: + if isinstance(value, str): + return [item.strip() for item in value.split(",") if item.strip()] + return value + + @field_validator("node_template", "edge_template", mode="before") + @classmethod + def parse_json_object(cls, value: object) -> object: + if isinstance(value, str): + parsed = json.loads(value) + if not isinstance(parsed, dict): + raise ValueError("template must be a JSON object") + return parsed + return value + + def model_definition(self) -> dict[str, Any]: + return { + "entity_types": self.entity_types, + "relationship_types": self.relationship_types, + "node_template": self.node_template, + "edge_template": self.edge_template, + "extraction_instructions": self.extraction_instructions, + } \ No newline at end of file diff --git a/src/graphrag_platform/llm.py b/src/graphrag_platform/llm.py new file mode 100644 index 0000000..8fe00ce --- /dev/null +++ b/src/graphrag_platform/llm.py @@ -0,0 +1,153 @@ +from __future__ import annotations + +import json +import logging +import random +import time +from collections.abc import Sequence +from typing import Any, Callable, Protocol + +import httpx + +from graphrag_platform.config import Settings + +LOGGER = logging.getLogger(__name__) +_RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504} + + +class LLMClient(Protocol): + chat_model: str + embedding_model: str + + def complete_json(self, system: str, user: str) -> dict[str, Any]: ... + + def complete_text(self, system: str, user: str) -> str: ... + + def embed(self, texts: Sequence[str]) -> list[list[float]]: ... + + +class OpenAICompatibleClient: + def __init__( + self, + settings: Settings, + transport: httpx.BaseTransport | None = None, + sleeper: Callable[[float], None] = time.sleep, + jitter: Callable[[], float] = random.random, + ) -> None: + if not settings.llm_api_key: + raise ValueError("LLM_API_KEY is required for LLM-backed stages") + self.chat_model = settings.llm_chat_model + self.embedding_model = settings.llm_embedding_model + self._settings = settings + self._sleeper = sleeper + self._jitter = jitter + self._client = httpx.Client( + base_url=settings.llm_base_url.rstrip("/") + "/", + timeout=settings.llm_timeout_seconds, + headers={ + "Authorization": f"Bearer {settings.llm_api_key}", + "Content-Type": "application/json", + }, + transport=transport, + ) + + def complete_json(self, system: str, user: str) -> dict[str, Any]: + response = self._post( + "chat/completions", + json={ + "model": self.chat_model, + "response_format": {"type": "json_object"}, + "temperature": 0, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + }, + ) + content = response.json()["choices"][0]["message"]["content"] + parsed = json.loads(content) + if not isinstance(parsed, dict): + raise ValueError("LLM response must be a JSON object") + return parsed + + def complete_text(self, system: str, user: str) -> str: + response = self._post( + "chat/completions", + json={ + "model": self.chat_model, + "temperature": 0, + "messages": [ + {"role": "system", "content": system}, + {"role": "user", "content": user}, + ], + }, + ) + return str(response.json()["choices"][0]["message"]["content"]).strip() + + def embed(self, texts: Sequence[str]) -> list[list[float]]: + if not texts: + return [] + response = self._post( + "embeddings", + json={"model": self.embedding_model, "input": list(texts)}, + ) + rows = sorted(response.json()["data"], key=lambda row: row["index"]) + return [[float(value) for value in row["embedding"]] for row in rows] + + def _post(self, path: str, *, json: dict[str, Any]) -> httpx.Response: + attempts = self._settings.llm_max_retries + 1 + for attempt in range(1, attempts + 1): + try: + response = self._client.post(path, json=json) + if response.status_code not in _RETRYABLE_STATUS_CODES: + response.raise_for_status() + return response + if attempt == attempts: + response.raise_for_status() + delay = self._retry_delay(response, attempt) + LOGGER.warning( + "Retrying model provider request", + extra={ + "event": "llm_retry", + "attempt": attempt, + "status_code": response.status_code, + "delay_seconds": delay, + "path": path, + }, + ) + self._sleeper(delay) + except (httpx.ConnectError, httpx.TimeoutException) as error: + if attempt == attempts: + raise + delay = self._retry_delay(None, attempt) + LOGGER.warning( + "Retrying model provider request after transport error", + extra={ + "event": "llm_retry", + "attempt": attempt, + "error_type": type(error).__name__, + "delay_seconds": delay, + "path": path, + }, + ) + self._sleeper(delay) + raise RuntimeError("model provider retry loop ended unexpectedly") + + def _retry_delay( + self, response: httpx.Response | None, attempt: int + ) -> float: + if response is not None: + retry_after = response.headers.get("Retry-After") + if retry_after: + try: + return min( + float(retry_after), + self._settings.llm_retry_max_seconds, + ) + except ValueError: + pass + exponential = self._settings.llm_retry_initial_seconds * (2 ** (attempt - 1)) + return min( + exponential + self._jitter(), + self._settings.llm_retry_max_seconds, + ) \ No newline at end of file diff --git a/src/graphrag_platform/models.py b/src/graphrag_platform/models.py new file mode 100644 index 0000000..ad219ad --- /dev/null +++ b/src/graphrag_platform/models.py @@ -0,0 +1,281 @@ +from __future__ import annotations + +from datetime import datetime, timezone +from typing import Any +from uuid import uuid4 + +from sqlalchemy import ( + DateTime, + Float, + ForeignKey, + Index, + Integer, + JSON, + String, + Text, + UniqueConstraint, +) +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column +from pgvector.sqlalchemy import Vector + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +class Base(DeclarativeBase): + pass + + +class RawDocument(Base): + __tablename__ = "raw_documents" + __table_args__ = ( + UniqueConstraint("corpus", "external_id", name="uq_raw_corpus_external_id"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + corpus: Mapped[str] = mapped_column(String(200), index=True) + external_id: Mapped[str] = mapped_column(String(500)) + source_schema_version: Mapped[str] = mapped_column(String(20)) + url: Mapped[str] = mapped_column(Text) + content_type: Mapped[str | None] = mapped_column(String(200)) + status_code: Mapped[int | None] = mapped_column(Integer) + body: Mapped[str] = mapped_column(Text) + metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) + collected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now) + + +class Document(Base): + __tablename__ = "documents" + __table_args__ = ( + UniqueConstraint("corpus", "url", name="uq_document_corpus_url"), + UniqueConstraint("raw_document_id", name="uq_document_raw_document"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + raw_document_id: Mapped[int | None] = mapped_column( + ForeignKey("raw_documents.id", ondelete="SET NULL") + ) + corpus: Mapped[str] = mapped_column(String(200), index=True) + url: Mapped[str] = mapped_column(Text) + title: Mapped[str | None] = mapped_column(Text) + author: Mapped[str | None] = mapped_column(Text) + published_at: Mapped[str | None] = mapped_column(String(100)) + text: Mapped[str] = mapped_column(Text) + content_hash: Mapped[str] = mapped_column(String(64), index=True) + metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + is_duplicate: Mapped[bool] = mapped_column(default=False) + canonical_document_id: Mapped[int | None] = mapped_column( + ForeignKey("documents.id", ondelete="SET NULL") + ) + + +class Chunk(Base): + __tablename__ = "chunks" + __table_args__ = (UniqueConstraint("document_id", "ordinal", name="uq_chunk_ordinal"),) + + id: Mapped[int] = mapped_column(primary_key=True) + document_id: Mapped[int] = mapped_column( + ForeignKey("documents.id", ondelete="CASCADE"), index=True + ) + corpus: Mapped[str] = mapped_column(String(200), index=True) + ordinal: Mapped[int] = mapped_column(Integer) + text: Mapped[str] = mapped_column(Text) + content_hash: Mapped[str] = mapped_column(String(64), index=True) + + +class KnowledgeModel(Base): + __tablename__ = "knowledge_models" + + corpus: Mapped[str] = mapped_column(String(200), primary_key=True) + definition_json: Mapped[dict[str, Any]] = mapped_column(JSON) + updated_at: Mapped[datetime] = mapped_column( + DateTime(timezone=True), default=utc_now, onupdate=utc_now + ) + + +class Extraction(Base): + __tablename__ = "extractions" + __table_args__ = (UniqueConstraint("chunk_id", name="uq_extraction_chunk"),) + + id: Mapped[int] = mapped_column(primary_key=True) + chunk_id: Mapped[int] = mapped_column( + ForeignKey("chunks.id", ondelete="CASCADE"), index=True + ) + corpus: Mapped[str] = mapped_column(String(200), index=True) + payload_json: Mapped[dict[str, Any]] = mapped_column(JSON) + model_name: Mapped[str] = mapped_column(String(200)) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now) + + +class ResolvedEntity(Base): + __tablename__ = "resolved_entities" + __table_args__ = ( + UniqueConstraint("corpus", "canonical_key", name="uq_resolved_entity_key"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + corpus: Mapped[str] = mapped_column(String(200), index=True) + canonical_key: Mapped[str] = mapped_column(String(500)) + canonical_name: Mapped[str] = mapped_column(Text) + entity_type: Mapped[str] = mapped_column(String(100)) + description: Mapped[str] = mapped_column(Text, default="") + aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list) + source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list) + + +class ResolvedRelationship(Base): + __tablename__ = "resolved_relationships" + __table_args__ = ( + UniqueConstraint( + "corpus", + "source_key", + "target_key", + "relationship_type", + name="uq_resolved_relationship", + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + corpus: Mapped[str] = mapped_column(String(200), index=True) + source_key: Mapped[str] = mapped_column(String(500)) + target_key: Mapped[str] = mapped_column(String(500)) + relationship_type: Mapped[str] = mapped_column(String(100)) + description: Mapped[str] = mapped_column(Text, default="") + source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list) + + +class Node(Base): + __tablename__ = "nodes" + __table_args__ = ( + UniqueConstraint("corpus", "canonical_key", name="uq_node_canonical_key"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + corpus: Mapped[str] = mapped_column(String(200), index=True) + canonical_key: Mapped[str] = mapped_column(String(500)) + canonical_name: Mapped[str] = mapped_column(Text) + entity_type: Mapped[str] = mapped_column(String(100), index=True) + description: Mapped[str] = mapped_column(Text, default="") + aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list) + source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list) + + +class Edge(Base): + __tablename__ = "edges" + __table_args__ = ( + UniqueConstraint( + "corpus", + "source_node_id", + "target_node_id", + "relationship_type", + name="uq_edge_identity", + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + corpus: Mapped[str] = mapped_column(String(200), index=True) + source_node_id: Mapped[int] = mapped_column( + ForeignKey("nodes.id", ondelete="CASCADE"), index=True + ) + target_node_id: Mapped[int] = mapped_column( + ForeignKey("nodes.id", ondelete="CASCADE"), index=True + ) + relationship_type: Mapped[str] = mapped_column(String(100), index=True) + description: Mapped[str] = mapped_column(Text, default="") + weight: Mapped[float] = mapped_column(Float, default=1.0) + source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list) + + +class Community(Base): + __tablename__ = "communities" + __table_args__ = (UniqueConstraint("corpus", "community_key", name="uq_community_key"),) + + id: Mapped[int] = mapped_column(primary_key=True) + corpus: Mapped[str] = mapped_column(String(200), index=True) + community_key: Mapped[str] = mapped_column(String(100)) + title: Mapped[str] = mapped_column(Text) + summary: Mapped[str] = mapped_column(Text) + node_ids_json: Mapped[list[int]] = mapped_column(JSON) + + +class SearchItem(Base): + __tablename__ = "retrieval_items" + __table_args__ = ( + UniqueConstraint("corpus", "item_type", "item_key", name="uq_search_item"), + Index("ix_retrieval_items_corpus_type", "corpus", "item_type"), + Index( + "ix_retrieval_items_corpus_model", + "corpus", + "embedding_model", + ), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + corpus: Mapped[str] = mapped_column(String(200), index=True) + item_type: Mapped[str] = mapped_column(String(50)) + item_key: Mapped[str] = mapped_column(String(100)) + text: Mapped[str] = mapped_column(Text) + source_url: Mapped[str | None] = mapped_column(Text) + metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + embedding: Mapped[list[float]] = mapped_column( + Vector().with_variant(JSON(), "sqlite") + ) + embedding_model: Mapped[str] = mapped_column(String(200)) + + +class PipelineRun(Base): + __tablename__ = "pipeline_runs" + __table_args__ = ( + Index("ix_pipeline_runs_status_created", "status", "created_at"), + Index("ix_pipeline_runs_corpus_created", "corpus", "created_at"), + ) + + id: Mapped[str] = mapped_column( + String(36), primary_key=True, default=lambda: str(uuid4()) + ) + corpus: Mapped[str] = mapped_column(String(200), index=True) + status: Mapped[str] = mapped_column(String(30), index=True, default="queued") + from_stage: Mapped[int] = mapped_column(Integer) + through_stage: Mapped[int] = mapped_column(Integer) + payload_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + configuration_hash: Mapped[str] = mapped_column(String(64)) + requested_by: Mapped[str] = mapped_column(String(500)) + correlation_id: Mapped[str] = mapped_column(String(200), index=True) + feature_name: Mapped[str] = mapped_column(String(200), default="KnowledgeGraph") + permission_mask: Mapped[int] = mapped_column(Integer) + attempt: Mapped[int] = mapped_column(Integer, default=0) + output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + error_message: Mapped[str | None] = mapped_column(Text) + created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now) + started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + + +class StageRun(Base): + __tablename__ = "stage_runs" + __table_args__ = ( + UniqueConstraint( + "pipeline_run_id", + "stage_number", + "attempt", + name="uq_stage_run_attempt", + ), + Index("ix_stage_runs_pipeline_stage", "pipeline_run_id", "stage_number"), + ) + + id: Mapped[int] = mapped_column(primary_key=True) + pipeline_run_id: Mapped[str] = mapped_column( + ForeignKey("pipeline_runs.id", ondelete="CASCADE"), index=True + ) + stage_number: Mapped[int] = mapped_column(Integer) + stage_name: Mapped[str] = mapped_column(String(200)) + status: Mapped[str] = mapped_column(String(30), default="running") + attempt: Mapped[int] = mapped_column(Integer) + input_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) + error_message: Mapped[str | None] = mapped_column(Text) + started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now) + finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) \ No newline at end of file diff --git a/src/graphrag_platform/observability.py b/src/graphrag_platform/observability.py new file mode 100644 index 0000000..785169e --- /dev/null +++ b/src/graphrag_platform/observability.py @@ -0,0 +1,33 @@ +from __future__ import annotations + +import json +import logging +from datetime import datetime, timezone +from typing import Any + + +class JsonFormatter(logging.Formatter): + _reserved = set(logging.makeLogRecord({}).__dict__) + + def format(self, record: logging.LogRecord) -> str: + payload: dict[str, Any] = { + "timestamp": datetime.now(timezone.utc).isoformat(), + "level": record.levelname, + "logger": record.name, + "message": record.getMessage(), + } + for key, value in record.__dict__.items(): + if key not in self._reserved and key not in {"message", "asctime"}: + payload[key] = value + if record.exc_info: + payload["exception"] = self.formatException(record.exc_info) + return json.dumps(payload, ensure_ascii=False, default=str) + + +def configure_logging(level: str = "INFO") -> None: + handler = logging.StreamHandler() + handler.setFormatter(JsonFormatter()) + root = logging.getLogger() + root.handlers.clear() + root.addHandler(handler) + root.setLevel(level.upper()) diff --git a/src/graphrag_platform/orchestrator.py b/src/graphrag_platform/orchestrator.py new file mode 100644 index 0000000..a3da801 --- /dev/null +++ b/src/graphrag_platform/orchestrator.py @@ -0,0 +1,330 @@ +from __future__ import annotations + +import hashlib +import json +import logging +from datetime import datetime, timezone +from pathlib import Path +from typing import Any + +from sqlalchemy import select, text, update + +from graphrag_platform.models import PipelineRun, StageRun +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import STAGE_MODULES, load_stage + +LOGGER = logging.getLogger(__name__) +BUILD_STAGES = range(1, 9) + + +class RunDeferredError(RuntimeError): + pass + + +def utc_now() -> datetime: + return datetime.now(timezone.utc) + + +def configuration_hash(context: StageContext) -> str: + payload = context.settings.model_dump( + mode="json", + exclude={"llm_api_key", "internal_jwt_secret"}, + ) + encoded = json.dumps(payload, sort_keys=True, separators=(",", ":")) + return hashlib.sha256(encoded.encode("utf-8")).hexdigest() + + +def submit_run( + context: StageContext, + *, + from_stage: int, + through_stage: int, + payload: dict[str, Any], + requested_by: str, + correlation_id: str, + permission_mask: int, +) -> str: + if from_stage not in BUILD_STAGES or through_stage not in BUILD_STAGES: + raise ValueError("pipeline jobs support build stages 1 through 8") + if from_stage > through_stage: + raise ValueError("from_stage must not exceed through_stage") + run = PipelineRun( + corpus=context.settings.corpus_name, + status="queued", + from_stage=from_stage, + through_stage=through_stage, + payload_json=payload, + configuration_hash=configuration_hash(context), + requested_by=requested_by, + correlation_id=correlation_id, + feature_name=context.settings.internal_feature_name, + permission_mask=permission_mask, + ) + with context.storage.session() as session: + session.add(run) + session.flush() + run_id = run.id + LOGGER.info( + "Pipeline run queued", + extra={ + "event": "pipeline_queued", + "run_id": run_id, + "corpus": context.settings.corpus_name, + "from_stage": from_stage, + "through_stage": through_stage, + "correlation_id": correlation_id, + }, + ) + return run_id + + +def claim_next_run(context: StageContext) -> str | None: + with context.storage.session() as session: + statement = ( + select(PipelineRun) + .where(PipelineRun.status == "queued") + .order_by(PipelineRun.created_at) + .limit(1) + ) + if context.storage.engine.dialect.name == "postgresql": + statement = statement.with_for_update(skip_locked=True) + run = session.scalar(statement) + if run is None: + return None + run.status = "running" + run.attempt += 1 + run.started_at = run.started_at or utc_now() + run.heartbeat_at = utc_now() + session.flush() + return run.id + + +def execute_run(context: StageContext, run_id: str) -> dict[str, Any]: + with context.storage.session() as session: + run = session.get(PipelineRun, run_id) + if run is None: + raise ValueError(f"pipeline run does not exist: {run_id}") + if run.status == "queued": + run.status = "running" + run.attempt += 1 + run.started_at = run.started_at or utc_now() + if run.status != "running": + raise ValueError(f"pipeline run {run_id} is {run.status}, not running") + from_stage = run.from_stage + through_stage = run.through_stage + payload = dict(run.payload_json) + attempt = run.attempt + execution_context = context.for_corpus(run.corpus) + corpus = run.corpus + + lock_connection = None + if context.storage.engine.dialect.name == "postgresql": + lock_connection = context.storage.engine.connect() + acquired = lock_connection.scalar( + text("SELECT pg_try_advisory_lock(hashtext(:key))"), + {"key": f"knowledgegraph:{corpus}"}, + ) + if not acquired: + lock_connection.close() + with context.storage.session() as session: + run = session.get(PipelineRun, run_id) + if run is not None: + run.status = "queued" + run.error_message = "Deferred because the corpus is already running" + raise RunDeferredError(f"corpus already has an active run: {corpus}") + + outputs: dict[str, Any] = {} + try: + if payload.get("reset") and attempt == 1: + execution_context.storage.clear_from_stage(corpus, from_stage) + for stage_number in range(from_stage, through_stage + 1): + previous_output = _successful_stage_output( + context, run_id, stage_number + ) + if previous_output is not None: + outputs[str(stage_number)] = previous_output + continue + stage_name = STAGE_MODULES[stage_number] + stage_run_id = _start_stage_run( + context, + run_id=run_id, + stage_number=stage_number, + stage_name=stage_name, + attempt=attempt, + payload=payload, + ) + try: + result = _execute_stage(execution_context, stage_number, payload) + except Exception as error: + _finish_stage_run( + context, + stage_run_id, + status="failed", + error_message=str(error), + ) + raise + _finish_stage_run( + context, + stage_run_id, + status="succeeded", + output=result, + ) + outputs[str(stage_number)] = result + _checkpoint(context, run_id, outputs) + with context.storage.session() as session: + run = session.get(PipelineRun, run_id) + if run is None: + raise RuntimeError(f"pipeline run disappeared: {run_id}") + run.status = "succeeded" + run.output_json = outputs + run.finished_at = utc_now() + run.heartbeat_at = utc_now() + LOGGER.info( + "Pipeline run succeeded", + extra={"event": "pipeline_succeeded", "run_id": run_id}, + ) + return outputs + except Exception as error: + with context.storage.session() as session: + run = session.get(PipelineRun, run_id) + if run is not None: + run.status = "failed" + run.error_message = str(error) + run.finished_at = utc_now() + run.heartbeat_at = utc_now() + LOGGER.exception( + "Pipeline run failed", + extra={"event": "pipeline_failed", "run_id": run_id}, + ) + raise + finally: + if lock_connection is not None: + lock_connection.execute( + text("SELECT pg_advisory_unlock(hashtext(:key))"), + {"key": f"knowledgegraph:{corpus}"}, + ) + lock_connection.close() + + +def requeue_failed_run(context: StageContext, run_id: str) -> None: + with context.storage.session() as session: + run = session.get(PipelineRun, run_id) + if run is None: + raise ValueError(f"pipeline run does not exist: {run_id}") + if run.status != "failed": + raise ValueError("only failed pipeline runs can be requeued") + run.status = "queued" + run.error_message = None + run.finished_at = None + + +def recover_stale_runs(context: StageContext) -> int: + threshold = utc_now().timestamp() - context.settings.worker_stale_run_seconds + cutoff = datetime.fromtimestamp(threshold, tz=timezone.utc) + with context.storage.session() as session: + result = session.execute( + update(PipelineRun) + .where( + PipelineRun.status == "running", + PipelineRun.heartbeat_at < cutoff, + ) + .values( + status="queued", + error_message="Recovered after a stale worker lease", + finished_at=None, + ) + ) + recovered = int(result.rowcount or 0) + if recovered: + LOGGER.warning( + "Recovered stale pipeline runs", + extra={"event": "pipeline_stale_recovered", "count": recovered}, + ) + return recovered + + +def _execute_stage( + context: StageContext, stage_number: int, payload: dict[str, Any] +) -> dict[str, Any]: + module = load_stage(stage_number) + if stage_number == 1: + input_path = payload.get("input_jsonl") + return module.run( + context, + input_jsonl=Path(input_path) if input_path else None, + ) + if stage_number == 3: + model_path = payload.get("model_file") + return module.run( + context, + model_file=Path(model_path) if model_path else None, + ) + return module.run(context) + + +def _start_stage_run( + context: StageContext, + *, + run_id: str, + stage_number: int, + stage_name: str, + attempt: int, + payload: dict[str, Any], +) -> int: + with context.storage.session() as session: + stage_run = StageRun( + pipeline_run_id=run_id, + stage_number=stage_number, + stage_name=stage_name, + status="running", + attempt=attempt, + input_json=payload, + ) + session.add(stage_run) + session.flush() + return stage_run.id + + +def _finish_stage_run( + context: StageContext, + stage_run_id: int, + *, + status: str, + output: dict[str, Any] | None = None, + error_message: str | None = None, +) -> None: + with context.storage.session() as session: + stage_run = session.get(StageRun, stage_run_id) + if stage_run is None: + raise RuntimeError(f"stage run disappeared: {stage_run_id}") + stage_run.status = status + stage_run.output_json = output or {} + stage_run.error_message = error_message + stage_run.finished_at = utc_now() + + +def _checkpoint( + context: StageContext, run_id: str, outputs: dict[str, Any] +) -> None: + with context.storage.session() as session: + run = session.get(PipelineRun, run_id) + if run is not None: + run.output_json = outputs + run.heartbeat_at = utc_now() + + +def _successful_stage_output( + context: StageContext, run_id: str, stage_number: int +) -> dict[str, Any] | None: + with context.storage.session() as session: + stage_run = session.scalar( + select(StageRun) + .where( + StageRun.pipeline_run_id == run_id, + StageRun.stage_number == stage_number, + StageRun.status == "succeeded", + ) + .order_by(StageRun.attempt.desc()) + .limit(1) + ) + return dict(stage_run.output_json) if stage_run is not None else None diff --git a/src/graphrag_platform/pipeline.py b/src/graphrag_platform/pipeline.py new file mode 100644 index 0000000..685ed51 --- /dev/null +++ b/src/graphrag_platform/pipeline.py @@ -0,0 +1,72 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Any + +from graphrag_platform.config import Settings +from graphrag_platform.llm import LLMClient +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import load_stage +from graphrag_platform.storage import Storage + + +class KnowledgeGraphPipeline: + """Compatibility facade over independently executable numbered stages.""" + + def __init__( + self, + settings: Settings, + *, + storage: Storage | None = None, + llm: LLMClient | None = None, + ) -> None: + self.context = StageContext.create(settings, storage=storage, llm=llm) + + @property + def settings(self) -> Settings: + return self.context.settings + + @property + def storage(self) -> Storage: + return self.context.storage + + def initialize(self) -> None: + self.context.initialize() + + def stage_01_normalize( + self, *, input_jsonl: Path | None = None + ) -> dict[str, int]: + return load_stage(1).run(self.context, input_jsonl=input_jsonl) + + def stage_02_clean_and_chunk(self) -> dict[str, int]: + return load_stage(2).run(self.context) + + def stage_03_define_model( + self, model_file: Path | None = None + ) -> dict[str, int]: + return load_stage(3).run(self.context, model_file=model_file) + + def stage_04_extract(self) -> dict[str, int]: + return load_stage(4).run(self.context) + + def stage_05_reconcile(self) -> dict[str, int]: + return load_stage(5).run(self.context) + + def stage_06_build_graph(self) -> dict[str, int]: + return load_stage(6).run(self.context) + + def stage_07_build_communities(self) -> dict[str, int]: + return load_stage(7).run(self.context) + + def stage_08_build_retrieval_index(self) -> dict[str, int]: + return load_stage(8).run(self.context) + + def stage_09_retrieve( + self, question: str, *, top_k: int | None = None + ) -> dict[str, Any]: + return load_stage(9).run(self.context, question=question, top_k=top_k) + + def stage_10_answer( + self, question: str, *, top_k: int | None = None + ) -> dict[str, Any]: + return load_stage(10).run(self.context, question=question, top_k=top_k) diff --git a/src/graphrag_platform/runtime.py b/src/graphrag_platform/runtime.py new file mode 100644 index 0000000..63e2676 --- /dev/null +++ b/src/graphrag_platform/runtime.py @@ -0,0 +1,50 @@ +from __future__ import annotations + +from dataclasses import dataclass + +from graphrag_platform.config import Settings +from graphrag_platform.llm import LLMClient, OpenAICompatibleClient +from graphrag_platform.storage import Storage + + +@dataclass +class StageContext: + settings: Settings + storage: Storage + _llm: LLMClient | None = None + + @classmethod + def create( + cls, + settings: Settings, + *, + storage: Storage | None = None, + llm: LLMClient | None = None, + ) -> StageContext: + return cls( + settings=settings, + storage=storage or Storage(settings.database_url), + _llm=llm, + ) + + @property + def llm(self) -> LLMClient: + if self._llm is None: + self._llm = OpenAICompatibleClient(self.settings) + return self._llm + + def initialize(self) -> None: + if self.settings.auto_create_schema: + self.storage.create_schema() + else: + self.storage.verify_schema() + (self.settings.artifact_dir / self.settings.corpus_name).mkdir( + parents=True, exist_ok=True + ) + + def for_corpus(self, corpus: str) -> StageContext: + return StageContext.create( + self.settings.model_copy(update={"corpus_name": corpus}), + storage=self.storage, + llm=self._llm, + ) diff --git a/src/graphrag_platform/sources.py b/src/graphrag_platform/sources.py new file mode 100644 index 0000000..e5b2557 --- /dev/null +++ b/src/graphrag_platform/sources.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import json +import re +from collections.abc import Iterator +from datetime import datetime +from pathlib import Path +from typing import Any, Protocol + +from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator +from sqlalchemy import MetaData, Table, create_engine, select + +_TABLE_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$") + + +class SourceRecord(BaseModel): + model_config = ConfigDict(extra="forbid") + + schema_version: str = "1.0" + external_id: str = Field(min_length=1, max_length=500) + corpus: str | None = Field(default=None, min_length=1, max_length=200) + url: HttpUrl + body: str = Field(min_length=1) + content_type: str = "text/html" + status_code: int = Field(default=200, ge=100, le=599) + retrieved_at: datetime + metadata: dict[str, Any] = Field(default_factory=dict) + + @field_validator("schema_version") + @classmethod + def require_supported_schema(cls, value: str) -> str: + if value != "1.0": + raise ValueError(f"unsupported source schema version: {value}") + return value + + +class SourceReader(Protocol): + def records(self) -> Iterator[SourceRecord]: ... + + +class JsonLinesSourceReader: + def __init__(self, path: Path) -> None: + self.path = path + + def records(self) -> Iterator[SourceRecord]: + with self.path.open("r", encoding="utf-8") as stream: + for line_number, line in enumerate(stream, start=1): + if not line.strip(): + continue + try: + payload = json.loads(line) + yield SourceRecord.model_validate(payload) + except (json.JSONDecodeError, ValueError) as error: + raise ValueError( + f"{self.path}:{line_number}: invalid source record: {error}" + ) from error + + +class PostgreSqlSourceReader: + def __init__(self, database_url: str, table_name: str) -> None: + if not database_url.startswith("postgresql"): + raise ValueError("SOURCE_DATABASE_URL must use PostgreSQL") + if not _TABLE_NAME_RE.fullmatch(table_name): + raise ValueError("SOURCE_TABLE_NAME must be an unqualified SQL identifier") + self.engine = create_engine(database_url, pool_pre_ping=True) + self.table_name = table_name + + def records(self) -> Iterator[SourceRecord]: + metadata = MetaData() + table = Table(self.table_name, metadata, autoload_with=self.engine) + required = { + "schema_version", + "external_id", + "url", + "body", + "retrieved_at", + } + missing = required - set(table.columns.keys()) + if missing: + raise ValueError( + f"source table {self.table_name} is missing columns: " + f"{', '.join(sorted(missing))}" + ) + with self.engine.connect() as connection: + for row in connection.execute(select(table).order_by(table.c.external_id)): + payload = { + key: value + for key, value in row._mapping.items() + if key in SourceRecord.model_fields + } + yield SourceRecord.model_validate(payload) + + +def source_reader( + *, + input_jsonl: Path | None, + source_database_url: str | None, + source_table_name: str, +) -> SourceReader: + if input_jsonl is not None and source_database_url: + raise ValueError("provide either --input-jsonl or SOURCE_DATABASE_URL, not both") + if input_jsonl is not None: + return JsonLinesSourceReader(input_jsonl) + if source_database_url: + return PostgreSqlSourceReader(source_database_url, source_table_name) + raise ValueError("provide --input-jsonl or configure SOURCE_DATABASE_URL") diff --git a/src/graphrag_platform/stage_loader.py b/src/graphrag_platform/stage_loader.py new file mode 100644 index 0000000..4a0a8c0 --- /dev/null +++ b/src/graphrag_platform/stage_loader.py @@ -0,0 +1,25 @@ +from __future__ import annotations + +from importlib import import_module +from types import ModuleType + +STAGE_MODULES = { + 1: "01_normalize_content", + 2: "02_clean_content", + 3: "03_define_knowledge_model", + 4: "04_extract_knowledge", + 5: "05_reconcile_knowledge", + 6: "06_build_knowledge_graph", + 7: "07_build_communities", + 8: "08_build_retrieval_index", + 9: "09_query_graphrag", + 10: "10_generate_answer", +} + + +def load_stage(stage: int) -> ModuleType: + try: + module_name = STAGE_MODULES[stage] + except KeyError as error: + raise ValueError(f"unsupported stage {stage}") from error + return import_module(module_name) diff --git a/src/graphrag_platform/storage.py b/src/graphrag_platform/storage.py new file mode 100644 index 0000000..b0d7e8e --- /dev/null +++ b/src/graphrag_platform/storage.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +from collections.abc import Iterator +from contextlib import contextmanager +from typing import Any, TypeVar + +from sqlalchemy import create_engine, delete, event, inspect, select, text +from sqlalchemy.engine import Engine +from sqlalchemy.orm import Session + +from graphrag_platform.models import ( + Base, + Chunk, + Community, + Document, + Edge, + Extraction, + KnowledgeModel, + Node, + RawDocument, + ResolvedEntity, + ResolvedRelationship, + SearchItem, +) + +ModelT = TypeVar("ModelT", bound=Base) + + +class Storage: + def __init__(self, database_url: str) -> None: + connect_args = {"check_same_thread": False} if database_url.startswith("sqlite") else {} + self.engine: Engine = create_engine( + database_url, + future=True, + pool_pre_ping=True, + connect_args=connect_args, + ) + if database_url.startswith("sqlite"): + event.listen(self.engine, "connect", self._enable_sqlite_foreign_keys) + + @staticmethod + def _enable_sqlite_foreign_keys( + dbapi_connection: Any, connection_record: Any + ) -> None: + del connection_record + cursor = dbapi_connection.cursor() + cursor.execute("PRAGMA foreign_keys=ON") + cursor.close() + + def create_schema(self) -> None: + if self.engine.dialect.name == "postgresql": + with self.engine.begin() as connection: + connection.execute(text("CREATE EXTENSION IF NOT EXISTS vector")) + Base.metadata.create_all(self.engine) + + def verify_schema(self) -> None: + tables = set(inspect(self.engine).get_table_names()) + required = {"pipeline_runs", "stage_runs", "documents", "retrieval_items"} + missing = required - tables + if missing: + raise RuntimeError( + "database migrations are required; missing tables: " + f"{', '.join(sorted(missing))}" + ) + if self.engine.dialect.name == "postgresql": + with self.engine.connect() as connection: + extension = connection.scalar( + text( + "SELECT extname FROM pg_extension " + "WHERE extname = 'vector'" + ) + ) + if extension != "vector": + raise RuntimeError("database migrations are required: vector extension") + + @property + def supports_vector_search(self) -> bool: + return self.engine.dialect.name == "postgresql" + + def ensure_vector_index(self, dimensions: int) -> None: + if not self.supports_vector_search: + return + index_name = f"ix_retrieval_items_embedding_hnsw_{dimensions}" + statement = f""" + CREATE INDEX IF NOT EXISTS {index_name} + ON retrieval_items + USING hnsw ((embedding::vector({dimensions})) vector_cosine_ops) + WHERE vector_dims(embedding) = {dimensions} + """ + with self.engine.begin() as connection: + connection.execute(text(statement)) + + @contextmanager + def session(self) -> Iterator[Session]: + with Session(self.engine) as session: + try: + yield session + session.commit() + except Exception: + session.rollback() + raise + + def upsert( + self, + session: Session, + model: type[ModelT], + identity: dict[str, Any], + values: dict[str, Any], + ) -> ModelT: + instance = session.scalar(select(model).filter_by(**identity)) + if instance is None: + instance = model(**identity, **values) + session.add(instance) + else: + for key, value in values.items(): + setattr(instance, key, value) + session.flush() + return instance + + def clear_from_stage(self, corpus: str, stage: int) -> None: + stage_tables: list[tuple[int, type[Base]]] = [ + (8, SearchItem), + (7, Community), + (6, Edge), + (6, Node), + (5, ResolvedRelationship), + (5, ResolvedEntity), + (4, Extraction), + (2, Chunk), + (1, RawDocument), + (1, Document), + ] + with self.session() as session: + for table_stage, model in stage_tables: + if table_stage >= stage: + session.execute(delete(model).where(model.corpus == corpus)) + if stage <= 3: + session.execute( + delete(KnowledgeModel).where(KnowledgeModel.corpus == corpus) + ) \ No newline at end of file diff --git a/src/graphrag_platform/worker.py b/src/graphrag_platform/worker.py new file mode 100644 index 0000000..cdb5e10 --- /dev/null +++ b/src/graphrag_platform/worker.py @@ -0,0 +1,61 @@ +from __future__ import annotations + +import argparse +import logging +import time + +from graphrag_platform.config import Settings +from graphrag_platform.observability import configure_logging +from graphrag_platform.orchestrator import ( + RunDeferredError, + claim_next_run, + execute_run, + recover_stale_runs, +) +from graphrag_platform.runtime import StageContext + +LOGGER = logging.getLogger(__name__) + + +def work_once(context: StageContext) -> bool: + run_id = claim_next_run(context) + if run_id is None: + return False + try: + execute_run(context, run_id) + except RunDeferredError: + LOGGER.info( + "Worker deferred a run for an active corpus", + extra={"event": "worker_job_deferred", "run_id": run_id}, + ) + except Exception: + LOGGER.exception( + "Worker execution failed", + extra={"event": "worker_job_failed", "run_id": run_id}, + ) + return True + + +def run_worker(context: StageContext, *, once: bool = False) -> None: + context.initialize() + recover_stale_runs(context) + while True: + worked = work_once(context) + if once: + return + if not worked: + time.sleep(context.settings.worker_poll_seconds) + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run the KnowledgeGraph job worker.") + parser.add_argument("--env-file", default=".env") + parser.add_argument("--once", action="store_true") + args = parser.parse_args() + settings = Settings(_env_file=args.env_file) + configure_logging(settings.log_level) + run_worker(StageContext.create(settings), once=args.once) + + +if __name__ == "__main__": + main() diff --git a/src/templates/knowledge_model.json b/src/templates/knowledge_model.json new file mode 100644 index 0000000..81209cb --- /dev/null +++ b/src/templates/knowledge_model.json @@ -0,0 +1,33 @@ +{ + "entity_types": [ + "Person", + "Organization", + "Product", + "Concept", + "Event", + "Location" + ], + "relationship_types": [ + "AFFILIATED_WITH", + "CREATED", + "DEPENDS_ON", + "MENTIONS", + "PART_OF", + "RELATED_TO" + ], + "node_template": { + "canonical_name": "string", + "entity_type": "one configured entity type", + "description": "source-grounded string", + "aliases": [ + "string" + ] + }, + "edge_template": { + "source": "entity name", + "target": "entity name", + "relationship_type": "one configured relationship type", + "description": "source-grounded string" + }, + "extraction_instructions": "Extract only entities and relationships explicitly supported by the supplied text." +} diff --git a/src/templates/mcp_documents.jsonl b/src/templates/mcp_documents.jsonl new file mode 100644 index 0000000..0d5ff8f --- /dev/null +++ b/src/templates/mcp_documents.jsonl @@ -0,0 +1 @@ +{"schema_version":"1.0","external_id":"docs-example-article","corpus":"product-knowledge","url":"https://docs.example.com/article","content_type":"text/html","status_code":200,"retrieved_at":"2026-01-01T00:00:00Z","body":"Article title

Article title

Article text collected with Web-Access.

","metadata":{"collector":"Web-Access","stored_by":"Data-Store"}} diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..c3066e7 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,58 @@ +from __future__ import annotations + +import hashlib +from collections.abc import Sequence +from typing import Any + + +class FakeLLM: + chat_model = "fake-chat-model" + embedding_model = "fake-embedding-model" + + def complete_json(self, system: str, user: str) -> dict[str, Any]: + if "community" in system.casefold(): + return { + "title": "Atlas product community", + "summary": "Acme created Atlas, a graph retrieval product.", + } + return { + "entities": [ + { + "canonical_name": "Acme", + "entity_type": "Organization", + "description": "Acme is a technology organization.", + "aliases": ["Acme Corp"], + }, + { + "canonical_name": "Atlas", + "entity_type": "Product", + "description": "Atlas is a graph retrieval product.", + "aliases": [], + }, + ], + "relationships": [ + { + "source": "Acme", + "target": "Atlas", + "relationship_type": "CREATED", + "description": "Acme created Atlas.", + } + ], + } + + def complete_text(self, system: str, user: str) -> str: + return "Acme created Atlas. [S1]" + + def embed(self, texts: Sequence[str]) -> list[list[float]]: + vectors: list[list[float]] = [] + for text in texts: + normalized = text.casefold() + digest = hashlib.sha256(normalized.encode("utf-8")).digest() + vectors.append( + [ + float("atlas" in normalized), + float("acme" in normalized), + digest[0] / 255, + ] + ) + return vectors diff --git a/tests/fixtures/external_documents.jsonl b/tests/fixtures/external_documents.jsonl new file mode 100644 index 0000000..134ff86 --- /dev/null +++ b/tests/fixtures/external_documents.jsonl @@ -0,0 +1,3 @@ +{"schema_version":"1.0","external_id":"atlas-overview","corpus":"acceptance-corpus","url":"https://docs.example/atlas","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:00:00Z","body":"Atlas Overview

Atlas

Acme Corporation created Atlas. Atlas provides graph retrieval for knowledge applications.

","metadata":{"collector":"Web-Access","stored_by":"Data-Store","tags":["product","graph"]}} +{"schema_version":"1.0","external_id":"atlas-architecture","corpus":"acceptance-corpus","url":"https://docs.example/atlas/architecture","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:01:00Z","body":"Atlas Architecture

Architecture

Acme Corp designed Atlas to combine a knowledge graph with vector retrieval. Atlas depends on PostgreSQL and PgVector.

","metadata":{"collector":"Web-Access","stored_by":"Data-Store","description":"Contains commas, \"quotes\", and aliases."}} +{"schema_version":"1.0","external_id":"atlas-overview-copy","corpus":"acceptance-corpus","url":"https://mirror.example/atlas","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:02:00Z","body":"Atlas Overview

Atlas

Acme Corporation created Atlas. Atlas provides graph retrieval for knowledge applications.

","metadata":{"collector":"Web-Access","stored_by":"Data-Store","duplicate_candidate":true}} diff --git a/tests/integration/test_api_worker.py b/tests/integration/test_api_worker.py new file mode 100644 index 0000000..c1b6424 --- /dev/null +++ b/tests/integration/test_api_worker.py @@ -0,0 +1,284 @@ +from __future__ import annotations + +import json +from datetime import datetime, timedelta, timezone +from pathlib import Path +from uuid import uuid4 + +import jwt +from fastapi.testclient import TestClient + +from graphrag_platform.api import create_app +from graphrag_platform.auth import Permission +from graphrag_platform.config import Settings +from graphrag_platform.runtime import StageContext +from graphrag_platform.worker import work_once + +from conftest import FakeLLM + + +def _token(settings: Settings, mask: int) -> str: + now = datetime.now(timezone.utc) + return jwt.encode( + { + "iss": settings.internal_jwt_issuer, + "aud": settings.internal_jwt_audience, + "sub": "dev-test-client", + "iat": now, + "exp": now + timedelta(minutes=10), + "jti": str(uuid4()), + "features": {settings.internal_feature_name: mask}, + }, + settings.internal_jwt_secret, + algorithm="HS256", + ) + + +def test_bff_contract_and_worker_execute_durable_job(tmp_path: Path) -> None: + source = tmp_path / "source.jsonl" + source.write_text( + json.dumps( + { + "schema_version": "1.0", + "external_id": "atlas", + "corpus": "api-corpus", + "url": "https://docs.example/atlas", + "retrieved_at": "2026-10-13T18:00:00Z", + "body": ( + "Atlas
" + "Acme created Atlas for graph retrieval.
" + ), + } + ) + + "\n", + encoding="utf-8", + ) + settings = Settings( + _env_file=None, + database_url=f"sqlite:///{tmp_path / 'api.db'}", + auto_create_schema=True, + artifact_dir=tmp_path / "artifacts", + llm_embedding_dimensions=3, + internal_jwt_secret="dev-test-secret-at-least-32-bytes-long", + ) + app = create_app(settings, llm=FakeLLM()) + full_token = _token( + settings, + int(Permission.READ | Permission.ADD | Permission.EXECUTE), + ) + + with TestClient(app) as client: + assert client.get("/internal/knowledge-graph/health").status_code == 200 + assert ( + client.post( + "/internal/knowledge-graph/jobs", + json={ + "corpus": "api-corpus", + "from_stage": 1, + "through_stage": 8, + "input_jsonl": str(source), + }, + ).status_code + == 401 + ) + execute_only = _token(settings, int(Permission.EXECUTE)) + assert ( + client.post( + "/internal/knowledge-graph/jobs", + headers={"Authorization": f"Bearer {execute_only}"}, + json={ + "corpus": "api-corpus", + "from_stage": 1, + "through_stage": 8, + "input_jsonl": str(source), + }, + ).status_code + == 403 + ) + queued = client.post( + "/internal/knowledge-graph/jobs", + headers={"Authorization": f"Bearer {full_token}"}, + json={ + "corpus": "api-corpus", + "from_stage": 1, + "through_stage": 8, + "input_jsonl": str(source), + }, + ) + assert queued.status_code == 202 + job_id = queued.json()["job_id"] + + assert work_once(StageContext.create(settings, llm=FakeLLM())) + + status_response = client.get( + f"/internal/knowledge-graph/jobs/{job_id}", + headers={"Authorization": f"Bearer {full_token}"}, + ) + assert status_response.status_code == 200 + assert status_response.json()["status"] == "succeeded" + assert len(status_response.json()["stages"]) == 8 + + answer = client.post( + "/internal/knowledge-graph/answer", + headers={"Authorization": f"Bearer {full_token}"}, + json={"corpus": "api-corpus", "question": "Who created Atlas?"}, + ) + assert answer.status_code == 200 + assert answer.json()["answer"] == "Acme created Atlas. [S1]" + + +def test_failed_job_can_be_repaired_and_retried(tmp_path: Path) -> None: + source = tmp_path / "retry-source.jsonl" + source.write_text( + json.dumps( + { + "schema_version": "2.0", + "external_id": "retry-record", + "corpus": "retry-corpus", + "url": "https://docs.example/retry", + "retrieved_at": "2026-10-13T18:00:00Z", + "body": "invalid schema", + } + ) + + "\n", + encoding="utf-8", + ) + settings = Settings( + _env_file=None, + database_url=f"sqlite:///{tmp_path / 'retry.db'}", + auto_create_schema=True, + artifact_dir=tmp_path / "artifacts", + llm_embedding_dimensions=3, + internal_jwt_secret="dev-test-secret-at-least-32-bytes-long", + ) + app = create_app(settings, llm=FakeLLM()) + token = _token( + settings, + int(Permission.READ | Permission.ADD | Permission.EXECUTE), + ) + headers = {"Authorization": f"Bearer {token}"} + + with TestClient(app) as client: + queued = client.post( + "/internal/knowledge-graph/jobs", + headers=headers, + json={ + "corpus": "retry-corpus", + "from_stage": 1, + "through_stage": 1, + "input_jsonl": str(source), + }, + ) + job_id = queued.json()["job_id"] + assert work_once(StageContext.create(settings, llm=FakeLLM())) + assert ( + client.get( + f"/internal/knowledge-graph/jobs/{job_id}", + headers=headers, + ).json()["status"] + == "failed" + ) + + source.write_text( + json.dumps( + { + "schema_version": "1.0", + "external_id": "retry-record", + "corpus": "retry-corpus", + "url": "https://docs.example/retry", + "retrieved_at": "2026-10-13T18:00:00Z", + "content_type": "text/plain", + "body": "Corrected source content.", + } + ) + + "\n", + encoding="utf-8", + ) + retried = client.post( + f"/internal/knowledge-graph/jobs/{job_id}/retry", + headers=headers, + ) + assert retried.status_code == 202 + assert work_once(StageContext.create(settings, llm=FakeLLM())) + status_response = client.get( + f"/internal/knowledge-graph/jobs/{job_id}", + headers=headers, + ) + assert status_response.json()["status"] == "succeeded" + assert status_response.json()["attempt"] == 2 + + +def test_retry_resumes_after_last_successful_stage(tmp_path: Path) -> None: + source = tmp_path / "resume-source.jsonl" + source.write_text( + json.dumps( + { + "schema_version": "1.0", + "external_id": "resume-record", + "corpus": "resume-corpus", + "url": "https://docs.example/resume", + "retrieved_at": "2026-10-13T18:00:00Z", + "content_type": "text/plain", + "body": "Acme created Atlas for graph retrieval.", + } + ) + + "\n", + encoding="utf-8", + ) + model_file = tmp_path / "model.json" + model_file.write_text("[]", encoding="utf-8") + settings = Settings( + _env_file=None, + database_url=f"sqlite:///{tmp_path / 'resume.db'}", + auto_create_schema=True, + artifact_dir=tmp_path / "artifacts", + llm_embedding_dimensions=3, + internal_jwt_secret="dev-test-secret-at-least-32-bytes-long", + ) + app = create_app(settings, llm=FakeLLM()) + token = _token( + settings, + int(Permission.READ | Permission.ADD | Permission.EXECUTE), + ) + headers = {"Authorization": f"Bearer {token}"} + + with TestClient(app) as client: + queued = client.post( + "/internal/knowledge-graph/jobs", + headers=headers, + json={ + "corpus": "resume-corpus", + "from_stage": 1, + "through_stage": 3, + "input_jsonl": str(source), + "model_file": str(model_file), + }, + ) + job_id = queued.json()["job_id"] + assert work_once(StageContext.create(settings, llm=FakeLLM())) + failed = client.get( + f"/internal/knowledge-graph/jobs/{job_id}", headers=headers + ).json() + assert failed["status"] == "failed" + + model_file.write_text( + json.dumps(settings.model_definition()), + encoding="utf-8", + ) + assert ( + client.post( + f"/internal/knowledge-graph/jobs/{job_id}/retry", + headers=headers, + ).status_code + == 202 + ) + assert work_once(StageContext.create(settings, llm=FakeLLM())) + succeeded = client.get( + f"/internal/knowledge-graph/jobs/{job_id}", headers=headers + ).json() + + assert succeeded["status"] == "succeeded" + stage_numbers = [row["stage"] for row in succeeded["stages"]] + assert stage_numbers.count(1) == 1 + assert stage_numbers.count(2) == 1 + assert stage_numbers.count(3) == 2 diff --git a/tests/integration/test_full_pipeline.py b/tests/integration/test_full_pipeline.py new file mode 100644 index 0000000..6cb007b --- /dev/null +++ b/tests/integration/test_full_pipeline.py @@ -0,0 +1,69 @@ +import json +from pathlib import Path + +from sqlalchemy import func, select + +from graphrag_platform.config import Settings +from graphrag_platform.models import Community, Edge, Node, SearchItem +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import load_stage + +from conftest import FakeLLM + + +def test_all_ten_stages_build_and_query_graph(tmp_path: Path) -> None: + input_file = tmp_path / "documents.jsonl" + input_file.write_text( + json.dumps( + { + "schema_version": "1.0", + "external_id": "atlas-doc", + "corpus": "atlas-corpus", + "url": "https://docs.example/atlas", + "retrieved_at": "2026-10-13T18:00:00Z", + "body": ( + "Atlas
" + "

Atlas

Acme created Atlas. Atlas provides graph " + "retrieval for knowledge applications.

" + ), + "metadata": {"collector": "Web-Access", "stored_by": "Data-Store"}, + } + ) + + "\n", + encoding="utf-8", + ) + settings = Settings( + _env_file=None, + database_url=f"sqlite:///{tmp_path / 'graph.db'}", + auto_create_schema=True, + artifact_dir=tmp_path / "artifacts", + corpus_name="atlas-corpus", + chunk_size=300, + chunk_overlap=30, + retrieval_top_k=5, + llm_embedding_dimensions=3, + ) + context = StageContext.create(settings, llm=FakeLLM()) + + assert load_stage(1).run(context, input_jsonl=input_file)["normalized"] == 1 + assert load_stage(2).run(context)["chunks"] == 1 + assert load_stage(3).run(context)["entity_types"] == 6 + assert load_stage(4).run(context)["extracted_chunks"] == 1 + assert load_stage(5).run(context) == {"entities": 2, "relationships": 1} + assert load_stage(6).run(context) == {"nodes": 2, "edges": 1} + assert load_stage(7).run(context)["communities"] == 1 + assert load_stage(8).run(context)["indexed_items"] == 5 + + retrieval = load_stage(9).run(context, question="Who created Atlas?") + response = load_stage(10).run(context, question="Who created Atlas?") + + assert retrieval["context"] + assert retrieval["graph_context"][0]["relationship"] == "CREATED" + assert retrieval["sources"][0]["url"] == "https://docs.example/atlas" + assert response["answer"] == "Acme created Atlas. [S1]" + assert response["sources"][0]["citation"] == "S1" + with context.storage.session() as session: + assert session.scalar(select(func.count()).select_from(Node)) == 2 + assert session.scalar(select(func.count()).select_from(Edge)) == 1 + assert session.scalar(select(func.count()).select_from(Community)) == 1 + assert session.scalar(select(func.count()).select_from(SearchItem)) == 5 diff --git a/tests/integration/test_pgvector_postgres.py b/tests/integration/test_pgvector_postgres.py new file mode 100644 index 0000000..33ede97 --- /dev/null +++ b/tests/integration/test_pgvector_postgres.py @@ -0,0 +1,140 @@ +from __future__ import annotations + +import os +from pathlib import Path + +import pytest +from sqlalchemy import text + +from graphrag_platform.config import Settings +from graphrag_platform.models import PipelineRun +from graphrag_platform.orchestrator import ( + RunDeferredError, + claim_next_run, + execute_run, + submit_run, +) +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import load_stage + +from conftest import FakeLLM + +POSTGRES_URL = os.getenv("TEST_POSTGRES_URL") + + +@pytest.mark.skipif( + not POSTGRES_URL, + reason="TEST_POSTGRES_URL is required for the PostgreSQL integration test", +) +def test_pgvector_index_and_graph_retrieval( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + settings = Settings( + _env_file=None, + database_url=POSTGRES_URL, + source_database_url=POSTGRES_URL, + source_table_name="external_documents", + artifact_dir=tmp_path / "artifacts", + corpus_name="pgvector-integration", + chunk_size=300, + chunk_overlap=30, + retrieval_top_k=5, + llm_embedding_dimensions=3, + ) + monkeypatch.setenv("DATABASE_URL", POSTGRES_URL) + from alembic import command + from alembic.config import Config + + alembic_config = Config("alembic.ini") + command.downgrade(alembic_config, "base") + command.upgrade(alembic_config, "head") + context = StageContext.create(settings, llm=FakeLLM()) + context.storage.clear_from_stage(settings.corpus_name, 1) + with context.storage.engine.begin() as connection: + connection.execute(text("DROP TABLE IF EXISTS external_documents")) + connection.execute( + text( + """ + CREATE TABLE external_documents ( + id bigserial PRIMARY KEY, + external_id text NOT NULL UNIQUE, + schema_version text NOT NULL, + corpus text, + url text NOT NULL, + body text NOT NULL, + content_type text NOT NULL, + status_code integer NOT NULL, + retrieved_at timestamptz NOT NULL, + metadata jsonb NOT NULL, + created_at timestamptz NOT NULL DEFAULT now() + ) + """ + ) + ) + connection.execute( + text( + """ + INSERT INTO external_documents ( + external_id, schema_version, corpus, url, body, + content_type, status_code, retrieved_at, metadata + ) VALUES ( + 'atlas-doc', '1.0', 'pgvector-integration', + 'https://docs.example/atlas', + 'Atlas

Atlas

Acme created Atlas. Atlas provides graph retrieval for knowledge applications.

', + 'text/html', 200, '2026-10-13T18:00:00Z', + '{"collector":"Web-Access","stored_by":"Data-Store"}'::jsonb + ) + """ + ) + ) + + load_stage(1).run(context) + load_stage(2).run(context) + load_stage(3).run(context) + load_stage(4).run(context) + load_stage(5).run(context) + load_stage(6).run(context) + load_stage(7).run(context) + load_stage(8).run(context) + result = load_stage(9).run(context, question="Who created Atlas?") + + assert result["context"] + assert result["graph_context"][0]["relationship"] == "CREATED" + assert result["sources"][0]["url"] == "https://docs.example/atlas" + with context.storage.engine.connect() as connection: + assert connection.scalar( + text("SELECT extversion FROM pg_extension WHERE extname = 'vector'") + ) + index_definition = connection.scalar( + text( + "SELECT indexdef FROM pg_indexes " + "WHERE indexname = 'ix_retrieval_items_embedding_hnsw_3'" + ) + ) + assert index_definition is not None + assert "USING hnsw" in index_definition + + run_id = submit_run( + context, + from_stage=2, + through_stage=2, + payload={}, + requested_by="integration-test", + correlation_id="pg-lock-test", + permission_mask=31, + ) + with context.storage.engine.connect() as lock_connection: + assert lock_connection.scalar( + text("SELECT pg_try_advisory_lock(hashtext(:key))"), + {"key": "knowledgegraph:pgvector-integration"}, + ) + with pytest.raises(RunDeferredError): + execute_run(context, run_id) + lock_connection.execute( + text("SELECT pg_advisory_unlock(hashtext(:key))"), + {"key": "knowledgegraph:pgvector-integration"}, + ) + with context.storage.session() as session: + assert session.get(PipelineRun, run_id).status == "queued" + assert claim_next_run(context) == run_id + assert execute_run(context, run_id)["2"]["chunks"] == 1 diff --git a/tests/performance/test_vector_query_load.py b/tests/performance/test_vector_query_load.py new file mode 100644 index 0000000..8b4d20c --- /dev/null +++ b/tests/performance/test_vector_query_load.py @@ -0,0 +1,64 @@ +from __future__ import annotations + +import os +import time + +import pytest + +from graphrag_platform.config import Settings +from graphrag_platform.models import SearchItem +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import load_stage + +from conftest import FakeLLM + +POSTGRES_URL = os.getenv("TEST_POSTGRES_URL") +RUN_PERFORMANCE_TESTS = os.getenv("RUN_PERFORMANCE_TESTS") == "1" + + +@pytest.mark.performance +@pytest.mark.skipif( + not POSTGRES_URL or not RUN_PERFORMANCE_TESTS, + reason="Set TEST_POSTGRES_URL and RUN_PERFORMANCE_TESTS=1", +) +def test_vector_query_latency_at_configured_load() -> None: + item_count = int(os.getenv("PERF_VECTOR_ITEM_COUNT", "10000")) + maximum_seconds = float(os.getenv("PERF_MAX_QUERY_SECONDS", "2.0")) + settings = Settings( + _env_file=None, + database_url=POSTGRES_URL, + corpus_name="performance-corpus", + llm_embedding_dimensions=3, + retrieval_top_k=10, + ) + context = StageContext.create(settings, llm=FakeLLM()) + context.initialize() + with context.storage.session() as session: + session.query(SearchItem).filter_by(corpus=settings.corpus_name).delete() + session.add_all( + SearchItem( + corpus=settings.corpus_name, + item_type="benchmark", + item_key=str(index), + text=f"Benchmark graph item {index}", + source_url=None, + metadata_json={}, + embedding=[ + float(index % 2), + float((index // 2) % 2), + (index % 255) / 255, + ], + embedding_model=FakeLLM.embedding_model, + ) + for index in range(item_count) + ) + context.storage.ensure_vector_index(3) + + started = time.perf_counter() + result = load_stage(9).run(context, question="Atlas graph retrieval", top_k=10) + elapsed = time.perf_counter() - started + + assert len(result["context"]) == 10 + assert elapsed <= maximum_seconds, ( + f"vector query took {elapsed:.3f}s; threshold is {maximum_seconds:.3f}s" + ) diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 0000000..4546db8 --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,25 @@ +from pathlib import Path + +from graphrag_platform.config import Settings + + +def test_settings_parse_runtime_templates(monkeypatch) -> None: + monkeypatch.setenv("ENTITY_TYPES", "Company,Product") + monkeypatch.setenv("RELATIONSHIP_TYPES", "CREATED,DEPENDS_ON") + monkeypatch.setenv("NODE_TEMPLATE", '{"name":"string"}') + + settings = Settings(_env_file=None) + + assert settings.entity_types == ["Company", "Product"] + assert settings.relationship_types == ["CREATED", "DEPENDS_ON"] + assert settings.node_template == {"name": "string"} + assert settings.database_url.startswith("postgresql+psycopg://") + + +def test_sqlite_is_available_as_explicit_local_override(tmp_path: Path) -> None: + settings = Settings( + _env_file=None, + database_url=f"sqlite:///{tmp_path / 'graph.db'}", + ) + + assert settings.database_url.startswith("sqlite:///") diff --git a/tests/unit/test_llm.py b/tests/unit/test_llm.py new file mode 100644 index 0000000..8933b05 --- /dev/null +++ b/tests/unit/test_llm.py @@ -0,0 +1,66 @@ +from __future__ import annotations + +import httpx + +from graphrag_platform.config import Settings +from graphrag_platform.llm import OpenAICompatibleClient + + +def test_llm_retries_rate_limit_and_honors_retry_after() -> None: + requests = 0 + delays: list[float] = [] + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal requests + requests += 1 + if requests == 1: + return httpx.Response( + 429, + headers={"Retry-After": "0.25"}, + json={"error": "rate limited"}, + request=request, + ) + return httpx.Response( + 200, + json={"choices": [{"message": {"content": "answer"}}]}, + request=request, + ) + + settings = Settings( + _env_file=None, + llm_api_key="test-key", + llm_max_retries=2, + ) + client = OpenAICompatibleClient( + settings, + transport=httpx.MockTransport(handler), + sleeper=delays.append, + jitter=lambda: 0, + ) + + assert client.complete_text("system", "user") == "answer" + assert requests == 2 + assert delays == [0.25] + + +def test_llm_does_not_retry_authentication_failure() -> None: + requests = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal requests + requests += 1 + return httpx.Response(401, json={"error": "unauthorized"}, request=request) + + client = OpenAICompatibleClient( + Settings(_env_file=None, llm_api_key="bad-key", llm_max_retries=3), + transport=httpx.MockTransport(handler), + sleeper=lambda _: None, + ) + + try: + client.complete_text("system", "user") + except httpx.HTTPStatusError: + pass + else: + raise AssertionError("Expected an HTTPStatusError") + assert requests == 1 diff --git a/tests/unit/test_pipeline.py b/tests/unit/test_pipeline.py new file mode 100644 index 0000000..c77d029 --- /dev/null +++ b/tests/unit/test_pipeline.py @@ -0,0 +1,113 @@ +from pathlib import Path + +import pytest +from sqlalchemy import select + +from graphrag_platform.config import Settings +from graphrag_platform.models import Chunk, Document, Extraction, ResolvedRelationship +from graphrag_platform.runtime import StageContext +from graphrag_platform.stage_loader import load_stage + +from conftest import FakeLLM + + +def test_chunk_overlap_must_be_smaller_than_chunk_size(tmp_path: Path) -> None: + settings = Settings( + _env_file=None, + database_url=f"sqlite:///{tmp_path / 'graph.db'}", + auto_create_schema=True, + artifact_dir=tmp_path / "artifacts", + chunk_size=200, + chunk_overlap=200, + ) + stage = load_stage(2) + + with pytest.raises(ValueError, match="CHUNK_OVERLAP"): + stage.chunk_text("word " * 100, size=200, overlap=200) + + +def test_reconciliation_keeps_cross_chunk_relationship(tmp_path: Path) -> None: + settings = Settings( + _env_file=None, + database_url=f"sqlite:///{tmp_path / 'graph.db'}", + auto_create_schema=True, + artifact_dir=tmp_path / "artifacts", + corpus_name="unit", + ) + context = StageContext.create(settings, llm=FakeLLM()) + context.initialize() + with context.storage.session() as session: + document = Document( + corpus="unit", + url="https://example.test/source", + text="Acme created Atlas.", + content_hash="document", + metadata_json={}, + ) + session.add(document) + session.flush() + first = Chunk( + document_id=document.id, + corpus="unit", + ordinal=0, + text="Acme created", + content_hash="first", + ) + second = Chunk( + document_id=document.id, + corpus="unit", + ordinal=1, + text="Atlas", + content_hash="second", + ) + session.add_all([first, second]) + session.flush() + session.add_all( + [ + Extraction( + chunk_id=first.id, + corpus="unit", + model_name="test", + payload_json={ + "entities": [ + { + "canonical_name": "Acme", + "entity_type": "Organization", + "aliases": [], + } + ], + "relationships": [ + { + "source": "Acme", + "target": "Atlas", + "relationship_type": "CREATED", + } + ], + }, + ), + Extraction( + chunk_id=second.id, + corpus="unit", + model_name="test", + payload_json={ + "entities": [ + { + "canonical_name": "Atlas", + "entity_type": "Product", + "aliases": [], + } + ], + "relationships": [], + }, + ), + ] + ) + + result = load_stage(5).run(context) + + assert result == {"entities": 2, "relationships": 1} + with context.storage.session() as session: + relationship = session.scalar(select(ResolvedRelationship)) + assert relationship is not None + assert relationship.source_key == "acme" + assert relationship.target_key == "atlas" diff --git a/tests/unit/test_sources.py b/tests/unit/test_sources.py new file mode 100644 index 0000000..462b244 --- /dev/null +++ b/tests/unit/test_sources.py @@ -0,0 +1,53 @@ +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from graphrag_platform.sources import JsonLinesSourceReader + + +def test_jsonl_source_contract_preserves_nested_metadata(tmp_path: Path) -> None: + source = tmp_path / "source.jsonl" + source.write_text( + json.dumps( + { + "schema_version": "1.0", + "external_id": "record-1", + "corpus": "test", + "url": "https://example.test/article", + "body": "
Text, with commas and \"quotes\".
", + "retrieved_at": "2026-10-13T18:00:00Z", + "metadata": {"tags": ["one", "two"], "nested": {"value": 1}}, + } + ) + + "\n", + encoding="utf-8", + ) + + records = list(JsonLinesSourceReader(source).records()) + + assert records[0].external_id == "record-1" + assert records[0].metadata["tags"] == ["one", "two"] + assert '"quotes"' in records[0].body + + +def test_jsonl_source_contract_rejects_unknown_schema(tmp_path: Path) -> None: + source = tmp_path / "source.jsonl" + source.write_text( + json.dumps( + { + "schema_version": "2.0", + "external_id": "record-1", + "url": "https://example.test/article", + "body": "text", + "retrieved_at": "2026-10-13T18:00:00Z", + } + ) + + "\n", + encoding="utf-8", + ) + + with pytest.raises(ValueError, match="unsupported source schema"): + list(JsonLinesSourceReader(source).records())