moved to this repo
This commit is contained in:
parent
95d328615d
commit
964b043a94
|
|
@ -0,0 +1,13 @@
|
|||
.env
|
||||
.git
|
||||
.github
|
||||
.venv
|
||||
__pycache__
|
||||
*.py[cod]
|
||||
.pytest_cache
|
||||
.coverage
|
||||
artifacts
|
||||
build
|
||||
dist
|
||||
*.egg-info
|
||||
tests
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
# Storage. Replace the host, user, password, and database for each environment.
|
||||
POSTGRES_PASSWORD=knowledgegraph-dev
|
||||
POSTGRES_PORT=5432
|
||||
API_PORT=8080
|
||||
DATABASE_URL=postgresql+psycopg://knowledgegraph_app:knowledgegraph-dev@localhost:5432/knowledgegraph
|
||||
SOURCE_DATABASE_URL=
|
||||
SOURCE_TABLE_NAME=external_documents
|
||||
ANALYTICS_DATABASE_URL=
|
||||
AUTO_CREATE_SCHEMA=false
|
||||
|
||||
# External handoff and corpus.
|
||||
CORPUS_NAME=product-knowledge
|
||||
|
||||
# Runtime model templates. JSON files passed with --model-file override these values.
|
||||
ENTITY_TYPES=Person,Organization,Product,Concept,Event,Location
|
||||
RELATIONSHIP_TYPES=AFFILIATED_WITH,CREATED,DEPENDS_ON,MENTIONS,PART_OF,RELATED_TO
|
||||
NODE_TEMPLATE={"canonical_name":"string","entity_type":"one configured entity type","description":"source-grounded string","aliases":["string"]}
|
||||
EDGE_TEMPLATE={"source":"entity name","target":"entity name","relationship_type":"one configured relationship type","description":"source-grounded string"}
|
||||
EXTRACTION_INSTRUCTIONS=Extract only entities and relationships explicitly supported by the supplied text.
|
||||
|
||||
# OpenAI-compatible chat and embedding provider.
|
||||
LLM_BASE_URL=https://api.openai.com/v1
|
||||
LLM_API_KEY=replace-with-provider-key
|
||||
LLM_CHAT_MODEL=gpt-4.1-mini
|
||||
LLM_EMBEDDING_MODEL=text-embedding-3-small
|
||||
LLM_EMBEDDING_DIMENSIONS=1536
|
||||
LLM_TIMEOUT_SECONDS=60
|
||||
LLM_MAX_RETRIES=4
|
||||
LLM_RETRY_INITIAL_SECONDS=1
|
||||
LLM_RETRY_MAX_SECONDS=30
|
||||
|
||||
# Processing and retrieval.
|
||||
CHUNK_SIZE=1200
|
||||
CHUNK_OVERLAP=150
|
||||
COMMUNITY_RESOLUTION=1.0
|
||||
RETRIEVAL_TOP_K=8
|
||||
RETRIEVAL_GRAPH_HOPS=2
|
||||
RETRIEVAL_GRAPH_LIMIT=50
|
||||
RETRIEVAL_HNSW_EF_SEARCH=100
|
||||
LOG_LEVEL=INFO
|
||||
INTERNAL_JWT_SECRET=replace-with-a-dev-test-secret-at-least-32-bytes
|
||||
INTERNAL_JWT_ISSUER=knowledgegraph-bff
|
||||
INTERNAL_JWT_AUDIENCE=knowledgegraph-service
|
||||
INTERNAL_FEATURE_NAME=KnowledgeGraph
|
||||
WORKER_POLL_SECONDS=2
|
||||
WORKER_STALE_RUN_SECONDS=3600
|
||||
ARTIFACT_DIR=artifacts
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
pull_request:
|
||||
|
||||
jobs:
|
||||
test:
|
||||
runs-on: ubuntu-latest
|
||||
services:
|
||||
postgres:
|
||||
image: pgvector/pgvector:pg16
|
||||
env:
|
||||
POSTGRES_USER: knowledgegraph_app
|
||||
POSTGRES_PASSWORD: test-password
|
||||
POSTGRES_DB: knowledgegraph
|
||||
ports:
|
||||
- 5432:5432
|
||||
options: >-
|
||||
--health-cmd "pg_isready -U knowledgegraph_app -d knowledgegraph"
|
||||
--health-interval 5s
|
||||
--health-timeout 5s
|
||||
--health-retries 20
|
||||
|
||||
env:
|
||||
DATABASE_URL: postgresql+psycopg://knowledgegraph_app:test-password@localhost:5432/knowledgegraph
|
||||
TEST_POSTGRES_URL: postgresql+psycopg://knowledgegraph_app:test-password@localhost:5432/knowledgegraph
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
cache: pip
|
||||
|
||||
- name: Install
|
||||
run: python -m pip install -e ".[test]"
|
||||
|
||||
- name: Compile
|
||||
run: python -m compileall -q src tests migrations
|
||||
|
||||
- name: Check dependencies
|
||||
run: python -m pip check
|
||||
|
||||
- name: Run portable tests
|
||||
run: python -m pytest -q --ignore=tests/integration/test_pgvector_postgres.py
|
||||
|
||||
- name: Run migration and PgVector integration
|
||||
run: python -m pytest tests/integration/test_pgvector_postgres.py -q
|
||||
|
||||
- name: Validate Compose
|
||||
run: docker compose config --quiet
|
||||
|
||||
- name: Build application image
|
||||
run: docker build -f Containerfile -t knowledgegraph:test .
|
||||
|
|
@ -1,414 +1,12 @@
|
|||
# ---> VisualStudio
|
||||
## Ignore Visual Studio temporary files, build results, and
|
||||
## files generated by popular Visual Studio add-ons.
|
||||
##
|
||||
## Get latest from https://github.com/github/gitignore/blob/main/VisualStudio.gitignore
|
||||
|
||||
# User-specific files
|
||||
*.rsuser
|
||||
*.suo
|
||||
*.user
|
||||
*.userosscache
|
||||
*.sln.docstates
|
||||
|
||||
# User-specific files (MonoDevelop/Xamarin Studio)
|
||||
*.userprefs
|
||||
|
||||
# Mono auto generated files
|
||||
mono_crash.*
|
||||
|
||||
# Build results
|
||||
[Dd]ebug/
|
||||
[Dd]ebugPublic/
|
||||
[Rr]elease/
|
||||
[Rr]eleases/
|
||||
x64/
|
||||
x86/
|
||||
[Ww][Ii][Nn]32/
|
||||
[Aa][Rr][Mm]/
|
||||
[Aa][Rr][Mm]64/
|
||||
bld/
|
||||
[Bb]in/
|
||||
[Oo]bj/
|
||||
[Ll]og/
|
||||
[Ll]ogs/
|
||||
|
||||
# Visual Studio 2015/2017 cache/options directory
|
||||
.vs/
|
||||
# Uncomment if you have tasks that create the project's static files in wwwroot
|
||||
#wwwroot/
|
||||
|
||||
# Visual Studio 2017 auto generated files
|
||||
Generated\ Files/
|
||||
|
||||
# MSTest test Results
|
||||
[Tt]est[Rr]esult*/
|
||||
[Bb]uild[Ll]og.*
|
||||
|
||||
# NUnit
|
||||
*.VisualState.xml
|
||||
TestResult.xml
|
||||
nunit-*.xml
|
||||
|
||||
# Build Results of an ATL Project
|
||||
[Dd]ebugPS/
|
||||
[Rr]eleasePS/
|
||||
dlldata.c
|
||||
|
||||
# Benchmark Results
|
||||
BenchmarkDotNet.Artifacts/
|
||||
|
||||
# .NET Core
|
||||
project.lock.json
|
||||
project.fragment.lock.json
|
||||
artifacts/
|
||||
|
||||
# ASP.NET Scaffolding
|
||||
ScaffoldingReadMe.txt
|
||||
|
||||
# StyleCop
|
||||
StyleCopReport.xml
|
||||
|
||||
# Files built by Visual Studio
|
||||
*_i.c
|
||||
*_p.c
|
||||
*_h.h
|
||||
*.ilk
|
||||
*.meta
|
||||
*.obj
|
||||
*.iobj
|
||||
*.pch
|
||||
*.pdb
|
||||
*.ipdb
|
||||
*.pgc
|
||||
*.pgd
|
||||
*.rsp
|
||||
*.sbr
|
||||
*.tlb
|
||||
*.tli
|
||||
*.tlh
|
||||
*.tmp
|
||||
*.tmp_proj
|
||||
*_wpftmp.csproj
|
||||
*.log
|
||||
*.tlog
|
||||
*.vspscc
|
||||
*.vssscc
|
||||
.builds
|
||||
*.pidb
|
||||
*.svclog
|
||||
*.scc
|
||||
|
||||
# Chutzpah Test files
|
||||
_Chutzpah*
|
||||
|
||||
# Visual C++ cache files
|
||||
ipch/
|
||||
*.aps
|
||||
*.ncb
|
||||
*.opendb
|
||||
*.opensdf
|
||||
*.sdf
|
||||
*.cachefile
|
||||
*.VC.db
|
||||
*.VC.VC.opendb
|
||||
|
||||
# Visual Studio profiler
|
||||
*.psess
|
||||
*.vsp
|
||||
*.vspx
|
||||
*.sap
|
||||
|
||||
# Visual Studio Trace Files
|
||||
*.e2e
|
||||
|
||||
# TFS 2012 Local Workspace
|
||||
$tf/
|
||||
|
||||
# Guidance Automation Toolkit
|
||||
*.gpState
|
||||
|
||||
# ReSharper is a .NET coding add-in
|
||||
_ReSharper*/
|
||||
*.[Rr]e[Ss]harper
|
||||
*.DotSettings.user
|
||||
|
||||
# TeamCity is a build add-in
|
||||
_TeamCity*
|
||||
|
||||
# DotCover is a Code Coverage Tool
|
||||
*.dotCover
|
||||
|
||||
# AxoCover is a Code Coverage Tool
|
||||
.axoCover/*
|
||||
!.axoCover/settings.json
|
||||
|
||||
# Coverlet is a free, cross platform Code Coverage Tool
|
||||
coverage*.json
|
||||
coverage*.xml
|
||||
coverage*.info
|
||||
|
||||
# Visual Studio code coverage results
|
||||
*.coverage
|
||||
*.coveragexml
|
||||
|
||||
# NCrunch
|
||||
_NCrunch_*
|
||||
.*crunch*.local.xml
|
||||
nCrunchTemp_*
|
||||
|
||||
# MightyMoose
|
||||
*.mm.*
|
||||
AutoTest.Net/
|
||||
|
||||
# Web workbench (sass)
|
||||
.sass-cache/
|
||||
|
||||
# Installshield output folder
|
||||
[Ee]xpress/
|
||||
|
||||
# DocProject is a documentation generator add-in
|
||||
DocProject/buildhelp/
|
||||
DocProject/Help/*.HxT
|
||||
DocProject/Help/*.HxC
|
||||
DocProject/Help/*.hhc
|
||||
DocProject/Help/*.hhk
|
||||
DocProject/Help/*.hhp
|
||||
DocProject/Help/Html2
|
||||
DocProject/Help/html
|
||||
|
||||
# Click-Once directory
|
||||
publish/
|
||||
|
||||
# Publish Web Output
|
||||
*.[Pp]ublish.xml
|
||||
*.azurePubxml
|
||||
# Note: Comment the next line if you want to checkin your web deploy settings,
|
||||
# but database connection strings (with potential passwords) will be unencrypted
|
||||
*.pubxml
|
||||
*.publishproj
|
||||
|
||||
# Microsoft Azure Web App publish settings. Comment the next line if you want to
|
||||
# checkin your Azure Web App publish settings, but sensitive information contained
|
||||
# in these scripts will be unencrypted
|
||||
PublishScripts/
|
||||
|
||||
# NuGet Packages
|
||||
*.nupkg
|
||||
# NuGet Symbol Packages
|
||||
*.snupkg
|
||||
# The packages folder can be ignored because of Package Restore
|
||||
**/[Pp]ackages/*
|
||||
# except build/, which is used as an MSBuild target.
|
||||
!**/[Pp]ackages/build/
|
||||
# Uncomment if necessary however generally it will be regenerated when needed
|
||||
#!**/[Pp]ackages/repositories.config
|
||||
# NuGet v3's project.json files produces more ignorable files
|
||||
*.nuget.props
|
||||
*.nuget.targets
|
||||
|
||||
# Microsoft Azure Build Output
|
||||
csx/
|
||||
*.build.csdef
|
||||
|
||||
# Microsoft Azure Emulator
|
||||
ecf/
|
||||
rcf/
|
||||
|
||||
# Windows Store app package directories and files
|
||||
AppPackages/
|
||||
BundleArtifacts/
|
||||
Package.StoreAssociation.xml
|
||||
_pkginfo.txt
|
||||
*.appx
|
||||
*.appxbundle
|
||||
*.appxupload
|
||||
|
||||
# Visual Studio cache files
|
||||
# files ending in .cache can be ignored
|
||||
*.[Cc]ache
|
||||
# but keep track of directories ending in .cache
|
||||
!?*.[Cc]ache/
|
||||
|
||||
# Others
|
||||
ClientBin/
|
||||
~$*
|
||||
*~
|
||||
*.dbmdl
|
||||
*.dbproj.schemaview
|
||||
*.jfm
|
||||
*.pfx
|
||||
*.publishsettings
|
||||
orleans.codegen.cs
|
||||
|
||||
# Including strong name files can present a security risk
|
||||
# (https://github.com/github/gitignore/pull/2483#issue-259490424)
|
||||
#*.snk
|
||||
|
||||
# Since there are multiple workflows, uncomment next line to ignore bower_components
|
||||
# (https://github.com/github/gitignore/pull/1529#issuecomment-104372622)
|
||||
#bower_components/
|
||||
|
||||
# RIA/Silverlight projects
|
||||
Generated_Code/
|
||||
|
||||
# Backup & report files from converting an old project file
|
||||
# to a newer Visual Studio version. Backup files are not needed,
|
||||
# because we have git ;-)
|
||||
_UpgradeReport_Files/
|
||||
Backup*/
|
||||
UpgradeLog*.XML
|
||||
UpgradeLog*.htm
|
||||
ServiceFabricBackup/
|
||||
*.rptproj.bak
|
||||
|
||||
# SQL Server files
|
||||
*.mdf
|
||||
*.ldf
|
||||
*.ndf
|
||||
|
||||
# Business Intelligence projects
|
||||
*.rdl.data
|
||||
*.bim.layout
|
||||
*.bim_*.settings
|
||||
*.rptproj.rsuser
|
||||
*- [Bb]ackup.rdl
|
||||
*- [Bb]ackup ([0-9]).rdl
|
||||
*- [Bb]ackup ([0-9][0-9]).rdl
|
||||
|
||||
# Microsoft Fakes
|
||||
FakesAssemblies/
|
||||
|
||||
# GhostDoc plugin setting file
|
||||
*.GhostDoc.xml
|
||||
|
||||
# Node.js Tools for Visual Studio
|
||||
.ntvs_analysis.dat
|
||||
node_modules/
|
||||
|
||||
# Visual Studio 6 build log
|
||||
*.plg
|
||||
|
||||
# Visual Studio 6 workspace options file
|
||||
*.opt
|
||||
|
||||
# Visual Studio 6 auto-generated workspace file (contains which files were open etc.)
|
||||
*.vbw
|
||||
|
||||
# Visual Studio 6 auto-generated project file (contains which files were open etc.)
|
||||
*.vbp
|
||||
|
||||
# Visual Studio 6 workspace and project file (working project files containing files to include in project)
|
||||
*.dsw
|
||||
*.dsp
|
||||
|
||||
# Visual Studio 6 technical files
|
||||
*.ncb
|
||||
*.aps
|
||||
|
||||
# Visual Studio LightSwitch build output
|
||||
**/*.HTMLClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/GeneratedArtifacts
|
||||
**/*.DesktopClient/ModelManifest.xml
|
||||
**/*.Server/GeneratedArtifacts
|
||||
**/*.Server/ModelManifest.xml
|
||||
_Pvt_Extensions
|
||||
|
||||
# Paket dependency manager
|
||||
.paket/paket.exe
|
||||
paket-files/
|
||||
|
||||
# FAKE - F# Make
|
||||
.fake/
|
||||
|
||||
# CodeRush personal settings
|
||||
.cr/personal
|
||||
|
||||
# Python Tools for Visual Studio (PTVS)
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
|
||||
# Cake - Uncomment if you are using it
|
||||
# tools/**
|
||||
# !tools/packages.config
|
||||
|
||||
# Tabs Studio
|
||||
*.tss
|
||||
|
||||
# Telerik's JustMock configuration file
|
||||
*.jmconfig
|
||||
|
||||
# BizTalk build output
|
||||
*.btp.cs
|
||||
*.btm.cs
|
||||
*.odx.cs
|
||||
*.xsd.cs
|
||||
|
||||
# OpenCover UI analysis results
|
||||
OpenCover/
|
||||
|
||||
# Azure Stream Analytics local run output
|
||||
ASALocalRun/
|
||||
|
||||
# MSBuild Binary and Structured Log
|
||||
*.binlog
|
||||
|
||||
# NVidia Nsight GPU debugger configuration file
|
||||
*.nvuser
|
||||
|
||||
# MFractors (Xamarin productivity tool) working folder
|
||||
.mfractor/
|
||||
|
||||
# Local History for Visual Studio
|
||||
.localhistory/
|
||||
|
||||
# Visual Studio History (VSHistory) files
|
||||
.vshistory/
|
||||
|
||||
# BeatPulse healthcheck temp database
|
||||
healthchecksdb
|
||||
|
||||
# Backup folder for Package Reference Convert tool in Visual Studio 2017
|
||||
MigrationBackup/
|
||||
|
||||
# Ionide (cross platform F# VS Code tools) working folder
|
||||
.ionide/
|
||||
|
||||
# Fody - auto-generated XML schema
|
||||
FodyWeavers.xsd
|
||||
|
||||
# VS Code files for those working on multiple tools
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
*.code-workspace
|
||||
|
||||
# Local History for Visual Studio Code
|
||||
.history/
|
||||
|
||||
# Windows Installer files from build outputs
|
||||
*.cab
|
||||
*.msi
|
||||
*.msix
|
||||
*.msm
|
||||
*.msp
|
||||
|
||||
# JetBrains Rider
|
||||
*.sln.iml
|
||||
|
||||
# ---> VisualStudioCode
|
||||
.vscode/*
|
||||
!.vscode/settings.json
|
||||
!.vscode/tasks.json
|
||||
!.vscode/launch.json
|
||||
!.vscode/extensions.json
|
||||
!.vscode/*.code-snippets
|
||||
|
||||
# Local History for Visual Studio Code
|
||||
.history/
|
||||
|
||||
# Built Visual Studio Code Extensions
|
||||
*.vsix
|
||||
|
||||
*.py[cod]
|
||||
.pytest_cache/
|
||||
.coverage
|
||||
htmlcov/
|
||||
.env
|
||||
*.db
|
||||
artifacts/
|
||||
dist/
|
||||
build/
|
||||
*.egg-info/
|
||||
|
|
@ -0,0 +1,26 @@
|
|||
FROM python:3.12-slim AS runtime
|
||||
|
||||
ENV PYTHONDONTWRITEBYTECODE=1 \
|
||||
PYTHONUNBUFFERED=1 \
|
||||
PIP_NO_CACHE_DIR=1
|
||||
|
||||
RUN groupadd --system knowledgegraph \
|
||||
&& useradd --system --gid knowledgegraph --create-home knowledgegraph
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY pyproject.toml README.md alembic.ini ./
|
||||
COPY migrations ./migrations
|
||||
COPY src ./src
|
||||
|
||||
RUN python -m pip install --upgrade pip \
|
||||
&& python -m pip install .
|
||||
|
||||
RUN mkdir -p /app/artifacts \
|
||||
&& chown -R knowledgegraph:knowledgegraph /app
|
||||
|
||||
USER knowledgegraph
|
||||
|
||||
EXPOSE 8080
|
||||
|
||||
CMD ["uvicorn", "graphrag_platform.api:app", "--host", "0.0.0.0", "--port", "8080"]
|
||||
642
README.md
642
README.md
|
|
@ -1,2 +1,642 @@
|
|||
# RAG-POC
|
||||
# KnowledgeGraph GraphRAG
|
||||
|
||||
An internal, restartable GraphRAG pipeline that normalizes externally collected
|
||||
content, extracts and reconciles knowledge, builds a graph, indexes graph and source
|
||||
evidence with PostgreSQL/PgVector, retrieves graph-aware context, and generates
|
||||
source-cited answers.
|
||||
|
||||
The collection process is outside this project. Existing **Web-Access MCP** and
|
||||
**Data-Store MCP** processes discover and persist source material. This project
|
||||
starts by validating and normalizing their versioned handoff.
|
||||
|
||||
## Architecture
|
||||
|
||||
```text
|
||||
Web-Access MCP + Data-Store MCP
|
||||
|
|
||||
| versioned JSONL or PostgreSQL handoff
|
||||
v
|
||||
01 normalize and parse
|
||||
v
|
||||
02 clean, deduplicate, and chunk
|
||||
v
|
||||
03 define knowledge model
|
||||
v
|
||||
04 extract entities and relationships
|
||||
v
|
||||
05 reconcile aliases and duplicates
|
||||
v
|
||||
06 materialize graph nodes and edges
|
||||
v
|
||||
07 detect and summarize communities
|
||||
v
|
||||
08 build PgVector HNSW retrieval index
|
||||
v
|
||||
09 retrieve vector + graph + source context
|
||||
v
|
||||
10 generate a cited answer
|
||||
```
|
||||
|
||||
PostgreSQL is the canonical checkpoint store. Every completed build stage persists
|
||||
its output, so later stages can be rebuilt without repeating valid upstream work.
|
||||
PgVector stores embeddings and performs server-side HNSW cosine search. Indexed
|
||||
relational edge tables provide bounded graph expansion.
|
||||
|
||||
The deployment consists of:
|
||||
|
||||
- PostgreSQL 16 with PgVector
|
||||
- Alembic migration job
|
||||
- Internal FastAPI service
|
||||
- PostgreSQL-backed asynchronous worker
|
||||
- Existing BFF as the policy enforcement point
|
||||
|
||||
The API is internal only and is not intended for direct customer access.
|
||||
|
||||
## Repository layout
|
||||
|
||||
| Path | Purpose |
|
||||
|---|---|
|
||||
| [`src/01_normalize_content.py`](src/01_normalize_content.py) through [`src/10_generate_answer.py`](src/10_generate_answer.py) | Complete numbered stage implementations |
|
||||
| [`src/graphrag_platform`](src/graphrag_platform) | Configuration, models, storage, orchestration, API, auth, worker, and provider clients |
|
||||
| [`migrations`](migrations) | Alembic database migrations |
|
||||
| [`tests`](tests) | Unit, API/worker, pipeline, migration, and real PgVector integration tests |
|
||||
| [`compose.yaml`](compose.yaml) | Reproducible PostgreSQL, migration, API, and worker environment |
|
||||
| [`Containerfile`](Containerfile) | Non-root application image |
|
||||
| [`.env.example`](.env.example) | Runtime configuration template |
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Python 3.11 or newer
|
||||
- Podman 5 or newer
|
||||
- `podman machine` running on Windows
|
||||
- An OpenAI-compatible chat and embeddings endpoint
|
||||
- External Web-Access MCP/Data-Store MCP collection process
|
||||
|
||||
## Quick start with Compose
|
||||
|
||||
Create the local environment:
|
||||
|
||||
```powershell
|
||||
Copy-Item .env.example .env
|
||||
```
|
||||
|
||||
Edit `.env` and set at minimum:
|
||||
|
||||
```dotenv
|
||||
POSTGRES_PASSWORD=choose-a-local-password
|
||||
DATABASE_URL=postgresql+psycopg://knowledgegraph_app:choose-a-local-password@localhost:5432/knowledgegraph
|
||||
|
||||
LLM_BASE_URL=https://api.openai.com/v1
|
||||
LLM_API_KEY=replace-with-provider-key
|
||||
LLM_CHAT_MODEL=gpt-4.1-mini
|
||||
LLM_EMBEDDING_MODEL=text-embedding-3-small
|
||||
LLM_EMBEDDING_DIMENSIONS=1536
|
||||
|
||||
INTERNAL_JWT_SECRET=replace-with-at-least-32-random-bytes
|
||||
```
|
||||
|
||||
`POSTGRES_PASSWORD` and the password in `DATABASE_URL` must match. Compose replaces
|
||||
the database host with `postgres` inside application containers.
|
||||
|
||||
Start the complete environment:
|
||||
|
||||
```powershell
|
||||
podman machine start
|
||||
podman compose up --detach --build
|
||||
```
|
||||
|
||||
Compose:
|
||||
|
||||
1. Starts `docker.io/pgvector/pgvector:pg16`.
|
||||
2. Waits for PostgreSQL readiness.
|
||||
3. Runs `alembic upgrade head`.
|
||||
4. Starts the API and worker.
|
||||
5. Waits for API dependency readiness.
|
||||
|
||||
Verify it:
|
||||
|
||||
```powershell
|
||||
podman compose ps
|
||||
Invoke-RestMethod http://127.0.0.1:8080/internal/knowledge-graph/health
|
||||
Invoke-RestMethod http://127.0.0.1:8080/internal/knowledge-graph/ready
|
||||
```
|
||||
|
||||
View logs:
|
||||
|
||||
```powershell
|
||||
podman compose logs --follow api worker
|
||||
```
|
||||
|
||||
Stop services while retaining PostgreSQL data:
|
||||
|
||||
```powershell
|
||||
podman compose down
|
||||
```
|
||||
|
||||
Delete services and all local database data:
|
||||
|
||||
```powershell
|
||||
podman compose down --volumes
|
||||
```
|
||||
|
||||
Volume removal is destructive.
|
||||
|
||||
## External source handoff
|
||||
|
||||
### JSON Lines contract
|
||||
|
||||
Each UTF-8 JSONL line must satisfy schema version `1.0`:
|
||||
|
||||
```json
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"external_id": "stable-source-record-id",
|
||||
"corpus": "product-knowledge",
|
||||
"url": "https://docs.example.com/article",
|
||||
"body": "<html><body><main>Article text</main></body></html>",
|
||||
"content_type": "text/html",
|
||||
"status_code": 200,
|
||||
"retrieved_at": "2026-10-13T18:00:00Z",
|
||||
"metadata": {
|
||||
"collector": "Web-Access",
|
||||
"stored_by": "Data-Store"
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Required fields:
|
||||
|
||||
- `schema_version`: currently `1.0`
|
||||
- `external_id`: stable and unique within a corpus
|
||||
- `url`
|
||||
- `body`
|
||||
- `retrieved_at`: ISO 8601 timestamp with offset
|
||||
|
||||
Optional/defaulted fields:
|
||||
|
||||
- `corpus`: when provided, it must match the selected runtime corpus
|
||||
- `content_type`: defaults to `text/html`
|
||||
- `status_code`: defaults to `200`
|
||||
- `metadata`: defaults to an empty object
|
||||
|
||||
Unknown fields and unsupported schema versions are rejected. Invalid records include
|
||||
the file and line number in the error.
|
||||
|
||||
The template is [`src/templates/mcp_documents.jsonl`](src/templates/mcp_documents.jsonl).
|
||||
A larger acceptance corpus is
|
||||
[`tests/fixtures/external_documents.jsonl`](tests/fixtures/external_documents.jsonl).
|
||||
|
||||
### PostgreSQL source contract
|
||||
|
||||
Set `SOURCE_DATABASE_URL` and `SOURCE_TABLE_NAME` to read the same contract from an
|
||||
external PostgreSQL table:
|
||||
|
||||
```dotenv
|
||||
SOURCE_DATABASE_URL=postgresql+psycopg://source_reader:password@source-host:5432/source_database
|
||||
SOURCE_TABLE_NAME=external_documents
|
||||
```
|
||||
|
||||
Compatible table:
|
||||
|
||||
```sql
|
||||
CREATE TABLE external_documents (
|
||||
external_id text PRIMARY KEY,
|
||||
schema_version text NOT NULL DEFAULT '1.0',
|
||||
corpus text,
|
||||
url text NOT NULL,
|
||||
body text NOT NULL,
|
||||
content_type text NOT NULL DEFAULT 'text/html',
|
||||
status_code integer NOT NULL DEFAULT 200,
|
||||
retrieved_at timestamptz NOT NULL,
|
||||
metadata jsonb NOT NULL DEFAULT '{}'::jsonb
|
||||
);
|
||||
```
|
||||
|
||||
Additional source-table columns are ignored. The application account should receive
|
||||
read-only access to the source table.
|
||||
|
||||
Provide either `--input-jsonl` or `SOURCE_DATABASE_URL`, not both.
|
||||
|
||||
## Durable checkpoints
|
||||
|
||||
Build stages persist these boundaries:
|
||||
|
||||
```text
|
||||
raw_documents + documents
|
||||
-> chunks
|
||||
-> knowledge_models
|
||||
-> extractions
|
||||
-> resolved_entities + resolved_relationships
|
||||
-> nodes + edges
|
||||
-> communities
|
||||
-> retrieval_items
|
||||
```
|
||||
|
||||
`pipeline_runs` and `stage_runs` separately record:
|
||||
|
||||
- queued/running/succeeded/failed state
|
||||
- selected stage range
|
||||
- attempt
|
||||
- configuration fingerprint
|
||||
- caller and correlation ID
|
||||
- stage inputs and outputs
|
||||
- timestamps and error details
|
||||
|
||||
The worker requeues runs abandoned beyond `WORKER_STALE_RUN_SECONDS`. Set this value
|
||||
above the expected maximum duration of one stage.
|
||||
|
||||
PostgreSQL advisory locks allow only one active build for a corpus. If another
|
||||
worker claims a job for that corpus, the job is returned to the queue instead of
|
||||
executing conflicting writes.
|
||||
|
||||
DataFrames are not stage checkpoints. A Pandas or Polars DataFrame may be used by a
|
||||
future adapter for one bounded transformation, but every completed result must be
|
||||
persisted before the next stage starts.
|
||||
|
||||
## Database migrations
|
||||
|
||||
Runtime services verify the migrated schema and do not silently mutate it.
|
||||
|
||||
Apply migrations:
|
||||
|
||||
```powershell
|
||||
alembic upgrade head
|
||||
```
|
||||
|
||||
Inspect the current revision:
|
||||
|
||||
```powershell
|
||||
alembic current
|
||||
alembic history
|
||||
```
|
||||
|
||||
Create a migration after changing SQLAlchemy models:
|
||||
|
||||
```powershell
|
||||
alembic revision --autogenerate -m "describe schema change"
|
||||
```
|
||||
|
||||
Review generated migrations before applying them. Compose runs upgrades through the
|
||||
one-shot `migrate` service before starting API and worker services.
|
||||
|
||||
`AUTO_CREATE_SCHEMA=true` exists only for isolated tests. Do not enable it in shared
|
||||
dev, test, or production environments.
|
||||
|
||||
## Run stages directly
|
||||
|
||||
Install a local editable environment:
|
||||
|
||||
```powershell
|
||||
py -3.12 -m venv .venv
|
||||
.\.venv\Scripts\Activate.ps1
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install -e ".[test]"
|
||||
```
|
||||
|
||||
Run migrations before the first command:
|
||||
|
||||
```powershell
|
||||
alembic upgrade head
|
||||
```
|
||||
|
||||
### 1. Normalize external content
|
||||
|
||||
```powershell
|
||||
python .\src\01_normalize_content.py `
|
||||
--corpus product-knowledge `
|
||||
--input-jsonl .\artifacts\mcp-documents.jsonl
|
||||
```
|
||||
|
||||
This validates the external handoff, stores raw provenance, runs Trafilatura for
|
||||
HTML, and persists normalized document records.
|
||||
|
||||
### 2. Clean, deduplicate, and chunk
|
||||
|
||||
```powershell
|
||||
python .\src\02_clean_content.py --corpus product-knowledge
|
||||
```
|
||||
|
||||
### 3. Define the knowledge model
|
||||
|
||||
```powershell
|
||||
python .\src\03_define_knowledge_model.py `
|
||||
--corpus product-knowledge `
|
||||
--model-file .\src\templates\knowledge_model.json
|
||||
```
|
||||
|
||||
### 4. Extract entities and relationships
|
||||
|
||||
```powershell
|
||||
python .\src\04_extract_knowledge.py --corpus product-knowledge
|
||||
```
|
||||
|
||||
LLM calls occur outside long database transactions. Each validated chunk result is
|
||||
committed independently.
|
||||
|
||||
### 5. Reconcile knowledge
|
||||
|
||||
```powershell
|
||||
python .\src\05_reconcile_knowledge.py --corpus product-knowledge
|
||||
```
|
||||
|
||||
### 6. Build graph nodes and edges
|
||||
|
||||
```powershell
|
||||
python .\src\06_build_knowledge_graph.py --corpus product-knowledge
|
||||
```
|
||||
|
||||
### 7. Build communities
|
||||
|
||||
```powershell
|
||||
python .\src\07_build_communities.py --corpus product-knowledge
|
||||
```
|
||||
|
||||
### 8. Build the PgVector index
|
||||
|
||||
```powershell
|
||||
python .\src\08_build_retrieval_index.py --corpus product-knowledge
|
||||
```
|
||||
|
||||
This embeds chunks, nodes, edges, and community summaries, validates dimensions,
|
||||
stores vectors, and creates a dimension-specific HNSW cosine index.
|
||||
|
||||
### 9. Retrieve GraphRAG context
|
||||
|
||||
```powershell
|
||||
python .\src\09_query_graphrag.py `
|
||||
--corpus product-knowledge `
|
||||
--question "Who created Atlas?" `
|
||||
--top-k 8
|
||||
```
|
||||
|
||||
The result contains vector `context`, expanded `graph_context`, and traceable
|
||||
`sources`.
|
||||
|
||||
### 10. Generate a cited answer
|
||||
|
||||
```powershell
|
||||
python .\src\10_generate_answer.py `
|
||||
--corpus product-knowledge `
|
||||
--question "Who created Atlas?"
|
||||
```
|
||||
|
||||
## Run a durable build
|
||||
|
||||
Run stages 1 through 8 under one durable pipeline record:
|
||||
|
||||
```powershell
|
||||
knowledgegraph run `
|
||||
--corpus product-knowledge `
|
||||
--from-stage 1 `
|
||||
--through-stage 8 `
|
||||
--input-jsonl .\artifacts\mcp-documents.jsonl `
|
||||
--model-file .\src\templates\knowledge_model.json `
|
||||
--requested-by local-developer
|
||||
```
|
||||
|
||||
Use `--from-stage` to rebuild only downstream state. The `--reset` option deletes
|
||||
the selected stage and downstream derived corpus data before execution.
|
||||
|
||||
## BFF authorization contract
|
||||
|
||||
The existing BFF authenticates callers and evaluates the `KnowledgeGraph` feature.
|
||||
Independent permission bits are:
|
||||
|
||||
```text
|
||||
Read = 1
|
||||
Add = 2
|
||||
Edit = 4
|
||||
Delete = 8
|
||||
Execute = 16
|
||||
```
|
||||
|
||||
| Operation | Required mask |
|
||||
|---|---:|
|
||||
| Read job/source/retrieval information | `Read` (`1`) |
|
||||
| Submit a stage-1 source job | `Add | Execute` (`18`) |
|
||||
| Run downstream build stages | `Execute` (`16`) |
|
||||
| Query or answer | `Read | Execute` (`17`) |
|
||||
| Retry a failed job | `Execute` (`16`) |
|
||||
|
||||
The BFF forwards a signed internal JWT. Required claims:
|
||||
|
||||
```json
|
||||
{
|
||||
"iss": "knowledgegraph-bff",
|
||||
"aud": "knowledgegraph-service",
|
||||
"sub": "internal-caller-id",
|
||||
"iat": 1791930000,
|
||||
"exp": 1791930600,
|
||||
"jti": "correlation-or-request-id",
|
||||
"features": {
|
||||
"KnowledgeGraph": 31
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Dev/test uses HS256 and `INTERNAL_JWT_SECRET`. Production deployments can replace
|
||||
this verifier with the BFF's workload identity, mTLS, or asymmetric/JWKS contract.
|
||||
Never trust unsigned permission headers.
|
||||
|
||||
## Internal API
|
||||
|
||||
| Endpoint | Purpose |
|
||||
|---|---|
|
||||
| `GET /internal/knowledge-graph/health` | Process liveness |
|
||||
| `GET /internal/knowledge-graph/ready` | Database and PgVector readiness |
|
||||
| `POST /internal/knowledge-graph/jobs` | Queue stages 1–8 |
|
||||
| `GET /internal/knowledge-graph/jobs/{jobId}` | Read durable job/stage status |
|
||||
| `POST /internal/knowledge-graph/jobs/{jobId}/retry` | Requeue a failed job |
|
||||
| `POST /internal/knowledge-graph/query` | Synchronous bounded retrieval |
|
||||
| `POST /internal/knowledge-graph/answer` | Synchronous cited answer |
|
||||
|
||||
Example job request:
|
||||
|
||||
```json
|
||||
{
|
||||
"corpus": "product-knowledge",
|
||||
"from_stage": 1,
|
||||
"through_stage": 8,
|
||||
"input_jsonl": "/app/artifacts/mcp-documents.jsonl",
|
||||
"model_file": "/app/src/templates/knowledge_model.json",
|
||||
"reset": false
|
||||
}
|
||||
```
|
||||
|
||||
Compose bind-mounts local [`artifacts`](artifacts) at `/app/artifacts`, allowing the
|
||||
worker to read JSONL handoffs placed there. In deployed environments, prefer the
|
||||
PostgreSQL source contract or a shared managed file volume.
|
||||
|
||||
Long-running stages return `202 Accepted` with a job ID. Query and answer requests
|
||||
remain synchronous and bounded.
|
||||
|
||||
## Retry and logging behavior
|
||||
|
||||
Model-provider calls retry:
|
||||
|
||||
- HTTP 429
|
||||
- HTTP 500, 502, 503, and 504
|
||||
- connection failures
|
||||
- timeouts
|
||||
|
||||
Retries honor numeric `Retry-After`, otherwise use bounded exponential backoff with
|
||||
jitter. Authentication and validation errors are not retried.
|
||||
|
||||
Logs are emitted as one JSON object per line and include event names, run IDs,
|
||||
corpus/stage context where available, attempts, status, and error details. Secrets
|
||||
are not logged.
|
||||
|
||||
## Tests
|
||||
|
||||
Portable tests:
|
||||
|
||||
```powershell
|
||||
pytest --ignore=.\tests\integration\test_pgvector_postgres.py
|
||||
```
|
||||
|
||||
Real PostgreSQL/PgVector migration and retrieval test:
|
||||
|
||||
```powershell
|
||||
$env:TEST_POSTGRES_URL = "postgresql+psycopg://knowledgegraph_app:password@localhost:5432/knowledgegraph"
|
||||
pytest .\tests\integration\test_pgvector_postgres.py
|
||||
Remove-Item Env:TEST_POSTGRES_URL
|
||||
```
|
||||
|
||||
The suite covers:
|
||||
|
||||
- Versioned source validation
|
||||
- Nested metadata and quoted/comma-containing content
|
||||
- Configuration parsing
|
||||
- Chunk validation
|
||||
- Cross-chunk relationship reconciliation
|
||||
- Provider retry policy
|
||||
- Full ten-stage behavior
|
||||
- Signed BFF authorization and permission failures
|
||||
- Durable API job and worker execution
|
||||
- Alembic migration against PostgreSQL
|
||||
- PgVector extension and HNSW index creation
|
||||
- Server-side vector retrieval, graph expansion, and citations
|
||||
|
||||
Opt-in vector load test:
|
||||
|
||||
```powershell
|
||||
$env:TEST_POSTGRES_URL = "postgresql+psycopg://knowledgegraph_app:password@localhost:5432/knowledgegraph"
|
||||
$env:RUN_PERFORMANCE_TESTS = "1"
|
||||
$env:PERF_VECTOR_ITEM_COUNT = "10000"
|
||||
$env:PERF_MAX_QUERY_SECONDS = "2.0"
|
||||
pytest .\tests\performance\test_vector_query_load.py
|
||||
```
|
||||
|
||||
Set the item count and latency threshold to the capacity target for the dev/test
|
||||
environment. Performance checks are excluded by default because machine resources
|
||||
vary.
|
||||
|
||||
CI is defined in [`.github/workflows/ci.yml`](.github/workflows/ci.yml) and compiles,
|
||||
tests, validates Compose, exercises real PgVector, and builds the application image.
|
||||
|
||||
## Persistent format policy
|
||||
|
||||
- PostgreSQL is the canonical operational store.
|
||||
- JSON Lines is the preferred file handoff/checkpoint format.
|
||||
- CSV is allowed only for explicitly defined flat import/export views.
|
||||
|
||||
CSV safely supports commas, quotes, and line breaks when written by a compliant CSV
|
||||
library. It remains unsuitable as the canonical graph format because aliases,
|
||||
metadata, source ID arrays, embeddings, and graph properties are nested and typed.
|
||||
Never construct CSV by joining strings with commas.
|
||||
|
||||
## RAG database alternatives
|
||||
|
||||
### PostgreSQL with PgVector
|
||||
|
||||
Implemented default. Choose it for SQL transactions, colocated provenance and
|
||||
vectors, self-managed deployment, and tunable HNSW retrieval. Graph traversal uses
|
||||
indexed relational edges rather than a native property-graph language.
|
||||
|
||||
Official reference: [PgVector](https://github.com/pgvector/pgvector).
|
||||
|
||||
### Azure Cosmos DB for NoSQL
|
||||
|
||||
Choose it for managed/serverless document storage, vector indexing, and first-party
|
||||
vector plus BM25 hybrid search. Cosmos DB's Gremlin API is separate from the NoSQL
|
||||
vector feature; use document adjacency or a separate graph store.
|
||||
|
||||
References:
|
||||
|
||||
- [Cosmos DB vector search](https://learn.microsoft.com/azure/cosmos-db/vector-search)
|
||||
- [Cosmos DB hybrid search](https://learn.microsoft.com/azure/cosmos-db/gen-ai/hybrid-search)
|
||||
- [Cosmos DB Gremlin API](https://learn.microsoft.com/azure/cosmos-db/gremlin/overview)
|
||||
|
||||
### Neo4j
|
||||
|
||||
Choose it when graph topology, paths, and multi-hop traversal are primary retrieval
|
||||
signals. Neo4j combines native property graphs, Cypher, vector indexes, and
|
||||
graph-aware hybrid retrieval. It requires a dedicated backend adapter; changing
|
||||
`DATABASE_URL` is not sufficient. Confirm version and edition requirements for the
|
||||
selected vector and unified-search features.
|
||||
|
||||
References:
|
||||
|
||||
- [Neo4j vector indexes](https://neo4j.com/docs/cypher-manual/current/indexes/search-indexes/vector-indexes/)
|
||||
- [Neo4j hybrid search](https://neo4j.com/developer/genai-ecosystem/hybrid-search/)
|
||||
|
||||
Other credible options:
|
||||
|
||||
- [Amazon Neptune Analytics](https://docs.aws.amazon.com/neptune-analytics/latest/userguide/what-is-neptune-analytics.html)
|
||||
for AWS-managed graph/vector analytics
|
||||
- [Azure AI Search](https://learn.microsoft.com/azure/search/hybrid-search-overview)
|
||||
as a secondary managed hybrid index without a native graph model
|
||||
|
||||
## Operational guidance
|
||||
|
||||
- Keep the API and worker on internal networks accessible only to the BFF and
|
||||
operators.
|
||||
- Replace all dev secrets before shared deployment.
|
||||
- Use a separate PostgreSQL role for migrations and a least-privilege runtime role.
|
||||
- Back up PostgreSQL before destructive resets or migrations.
|
||||
- Monitor queued/running/failed run counts, stage duration, LLM retries, database
|
||||
pool usage, vector latency, and retrieval result counts.
|
||||
- Set `WORKER_STALE_RUN_SECONDS` above the longest expected single stage.
|
||||
- Rebuild stage 8 after changing embedding model or dimensions.
|
||||
- Maintain a golden question/citation set to detect retrieval-quality regressions.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Database migrations are required
|
||||
|
||||
```powershell
|
||||
alembic current
|
||||
alembic upgrade head
|
||||
```
|
||||
|
||||
### `LLM_API_KEY is required`
|
||||
|
||||
Set `LLM_API_KEY` in `.env` and restart API/worker containers.
|
||||
|
||||
### Embedding dimension mismatch
|
||||
|
||||
Set `LLM_EMBEDDING_DIMENSIONS` to the exact provider output dimension, then reset
|
||||
and rerun stage 8.
|
||||
|
||||
### Source handoff rejected
|
||||
|
||||
Validate every JSONL line against schema version `1.0`. Confirm `corpus`,
|
||||
`external_id`, URL, body, and `retrieved_at`.
|
||||
|
||||
### Job remains queued
|
||||
|
||||
```powershell
|
||||
podman compose ps worker
|
||||
podman compose logs --tail 100 worker
|
||||
```
|
||||
|
||||
### PostgreSQL/PgVector is not ready
|
||||
|
||||
```powershell
|
||||
podman compose ps postgres
|
||||
podman compose logs --tail 100 postgres migrate
|
||||
```
|
||||
|
||||
### Port conflict
|
||||
|
||||
Change `POSTGRES_PORT` or `API_PORT` in `.env`, then recreate the Compose stack.
|
||||
|
|
|
|||
|
|
@ -0,0 +1,38 @@
|
|||
[alembic]
|
||||
script_location = migrations
|
||||
prepend_sys_path = .
|
||||
path_separator = os
|
||||
|
||||
[loggers]
|
||||
keys = root,sqlalchemy,alembic
|
||||
|
||||
[handlers]
|
||||
keys = console
|
||||
|
||||
[formatters]
|
||||
keys = generic
|
||||
|
||||
[logger_root]
|
||||
level = WARN
|
||||
handlers = console
|
||||
qualname =
|
||||
|
||||
[logger_sqlalchemy]
|
||||
level = WARN
|
||||
handlers =
|
||||
qualname = sqlalchemy.engine
|
||||
|
||||
[logger_alembic]
|
||||
level = INFO
|
||||
handlers =
|
||||
qualname = alembic
|
||||
|
||||
[handler_console]
|
||||
class = StreamHandler
|
||||
args = (sys.stderr,)
|
||||
level = NOTSET
|
||||
formatter = generic
|
||||
|
||||
[formatter_generic]
|
||||
format = %(levelname)-5.5s [%(name)s] %(message)s
|
||||
datefmt = %H:%M:%S
|
||||
|
|
@ -0,0 +1,99 @@
|
|||
name: knowledgegraph
|
||||
|
||||
services:
|
||||
postgres:
|
||||
image: docker.io/pgvector/pgvector:pg16
|
||||
environment:
|
||||
POSTGRES_USER: knowledgegraph_app
|
||||
POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-knowledgegraph-dev}
|
||||
POSTGRES_DB: knowledgegraph
|
||||
ports:
|
||||
- "127.0.0.1:${POSTGRES_PORT:-5432}:5432"
|
||||
volumes:
|
||||
- knowledgegraph-postgres-data:/var/lib/postgresql/data
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD-SHELL",
|
||||
"pg_isready -U knowledgegraph_app -d knowledgegraph",
|
||||
]
|
||||
interval: 5s
|
||||
timeout: 5s
|
||||
retries: 20
|
||||
restart: unless-stopped
|
||||
|
||||
migrate:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Containerfile
|
||||
command: ["alembic", "upgrade", "head"]
|
||||
environment: &application-environment
|
||||
DATABASE_URL: postgresql+psycopg://knowledgegraph_app:${POSTGRES_PASSWORD:-knowledgegraph-dev}@postgres:5432/knowledgegraph
|
||||
SOURCE_DATABASE_URL: ${SOURCE_DATABASE_URL:-}
|
||||
SOURCE_TABLE_NAME: ${SOURCE_TABLE_NAME:-external_documents}
|
||||
AUTO_CREATE_SCHEMA: "false"
|
||||
CORPUS_NAME: ${CORPUS_NAME:-default-corpus}
|
||||
LLM_BASE_URL: ${LLM_BASE_URL:-https://api.openai.com/v1}
|
||||
LLM_API_KEY: ${LLM_API_KEY:-}
|
||||
LLM_CHAT_MODEL: ${LLM_CHAT_MODEL:-gpt-4.1-mini}
|
||||
LLM_EMBEDDING_MODEL: ${LLM_EMBEDDING_MODEL:-text-embedding-3-small}
|
||||
LLM_EMBEDDING_DIMENSIONS: ${LLM_EMBEDDING_DIMENSIONS:-1536}
|
||||
INTERNAL_JWT_SECRET: ${INTERNAL_JWT_SECRET:-knowledgegraph-dev-secret-change-this-32}
|
||||
INTERNAL_JWT_ISSUER: ${INTERNAL_JWT_ISSUER:-knowledgegraph-bff}
|
||||
INTERNAL_JWT_AUDIENCE: ${INTERNAL_JWT_AUDIENCE:-knowledgegraph-service}
|
||||
INTERNAL_FEATURE_NAME: ${INTERNAL_FEATURE_NAME:-KnowledgeGraph}
|
||||
LOG_LEVEL: ${LOG_LEVEL:-INFO}
|
||||
depends_on:
|
||||
postgres:
|
||||
condition: service_healthy
|
||||
restart: "no"
|
||||
|
||||
api:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Containerfile
|
||||
command:
|
||||
[
|
||||
"uvicorn",
|
||||
"graphrag_platform.api:app",
|
||||
"--host",
|
||||
"0.0.0.0",
|
||||
"--port",
|
||||
"8080",
|
||||
]
|
||||
environment: *application-environment
|
||||
ports:
|
||||
- "127.0.0.1:${API_PORT:-8080}:8080"
|
||||
volumes:
|
||||
- ./artifacts:/app/artifacts
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
healthcheck:
|
||||
test:
|
||||
[
|
||||
"CMD",
|
||||
"python",
|
||||
"-c",
|
||||
"import urllib.request; urllib.request.urlopen('http://127.0.0.1:8080/internal/knowledge-graph/ready', timeout=5)",
|
||||
]
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 12
|
||||
restart: unless-stopped
|
||||
|
||||
worker:
|
||||
build:
|
||||
context: .
|
||||
dockerfile: Containerfile
|
||||
command: ["knowledgegraph-worker"]
|
||||
environment: *application-environment
|
||||
volumes:
|
||||
- ./artifacts:/app/artifacts
|
||||
depends_on:
|
||||
migrate:
|
||||
condition: service_completed_successfully
|
||||
restart: unless-stopped
|
||||
|
||||
volumes:
|
||||
knowledgegraph-postgres-data:
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from logging.config import fileConfig
|
||||
|
||||
from alembic import context
|
||||
from sqlalchemy import engine_from_config, pool
|
||||
|
||||
from graphrag_platform.config import Settings
|
||||
from graphrag_platform.models import Base
|
||||
|
||||
config = context.config
|
||||
if config.config_file_name:
|
||||
fileConfig(config.config_file_name)
|
||||
|
||||
config.set_main_option("sqlalchemy.url", Settings().database_url)
|
||||
target_metadata = Base.metadata
|
||||
|
||||
|
||||
def run_migrations_offline() -> None:
|
||||
context.configure(
|
||||
url=config.get_main_option("sqlalchemy.url"),
|
||||
target_metadata=target_metadata,
|
||||
literal_binds=True,
|
||||
dialect_opts={"paramstyle": "named"},
|
||||
compare_type=True,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
def run_migrations_online() -> None:
|
||||
connectable = engine_from_config(
|
||||
config.get_section(config.config_ini_section, {}),
|
||||
prefix="sqlalchemy.",
|
||||
poolclass=pool.NullPool,
|
||||
)
|
||||
with connectable.connect() as connection:
|
||||
context.configure(
|
||||
connection=connection,
|
||||
target_metadata=target_metadata,
|
||||
compare_type=True,
|
||||
)
|
||||
with context.begin_transaction():
|
||||
context.run_migrations()
|
||||
|
||||
|
||||
if context.is_offline_mode():
|
||||
run_migrations_offline()
|
||||
else:
|
||||
run_migrations_online()
|
||||
|
|
@ -0,0 +1,24 @@
|
|||
"""${message}
|
||||
|
||||
Revision ID: ${up_revision}
|
||||
Revises: ${down_revision | comma,n}
|
||||
Create Date: ${create_date}
|
||||
"""
|
||||
from typing import Sequence, Union
|
||||
|
||||
from alembic import op
|
||||
import sqlalchemy as sa
|
||||
${imports if imports else ""}
|
||||
|
||||
revision: str = ${repr(up_revision)}
|
||||
down_revision: Union[str, None] = ${repr(down_revision)}
|
||||
branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)}
|
||||
depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)}
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
${upgrades if upgrades else "pass"}
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
${downgrades if downgrades else "pass"}
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
"""Create the initial KnowledgeGraph schema.
|
||||
|
||||
Revision ID: 20261013_0001
|
||||
Revises:
|
||||
Create Date: 2026-10-13
|
||||
"""
|
||||
|
||||
from alembic import op
|
||||
|
||||
from graphrag_platform.models import Base
|
||||
|
||||
revision = "20261013_0001"
|
||||
down_revision = None
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
connection = op.get_bind()
|
||||
connection.exec_driver_sql("CREATE EXTENSION IF NOT EXISTS vector")
|
||||
Base.metadata.create_all(bind=connection)
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
Base.metadata.drop_all(bind=op.get_bind())
|
||||
|
|
@ -0,0 +1,60 @@
|
|||
[build-system]
|
||||
requires = ["hatchling>=1.25"]
|
||||
build-backend = "hatchling.build"
|
||||
|
||||
[project]
|
||||
name = "knowledgegraph-graphrag"
|
||||
version = "0.1.0"
|
||||
description = "Production-oriented KnowledgeGraph ingestion, indexing, retrieval, and answer pipeline"
|
||||
readme = "README.md"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"alembic>=1.13,<2",
|
||||
"fastapi>=0.115,<1",
|
||||
"httpx>=0.27,<1",
|
||||
"networkx>=3.3,<4",
|
||||
"pgvector>=0.3,<1",
|
||||
"PyJWT>=2.9,<3",
|
||||
"pydantic>=2.8,<3",
|
||||
"pydantic-settings>=2.4,<3",
|
||||
"psycopg[binary]>=3.2,<4",
|
||||
"sqlalchemy>=2.0,<3",
|
||||
"trafilatura>=1.12,<3",
|
||||
"uvicorn[standard]>=0.32,<1",
|
||||
]
|
||||
|
||||
[project.optional-dependencies]
|
||||
test = [
|
||||
"pytest>=8.3,<9",
|
||||
"pytest-cov>=5,<7",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
knowledgegraph = "graphrag_platform.cli:main_all"
|
||||
knowledgegraph-worker = "graphrag_platform.worker:main"
|
||||
|
||||
[tool.hatch.build.targets.wheel]
|
||||
packages = [
|
||||
"src/graphrag_platform",
|
||||
"src/01_normalize_content.py",
|
||||
"src/02_clean_content.py",
|
||||
"src/03_define_knowledge_model.py",
|
||||
"src/04_extract_knowledge.py",
|
||||
"src/05_reconcile_knowledge.py",
|
||||
"src/06_build_knowledge_graph.py",
|
||||
"src/07_build_communities.py",
|
||||
"src/08_build_retrieval_index.py",
|
||||
"src/09_query_graphrag.py",
|
||||
"src/10_generate_answer.py",
|
||||
]
|
||||
|
||||
[tool.pytest.ini_options]
|
||||
addopts = "-ra"
|
||||
testpaths = ["tests"]
|
||||
markers = [
|
||||
"performance: opt-in performance and load checks",
|
||||
]
|
||||
|
||||
[tool.coverage.run]
|
||||
source = ["graphrag_platform"]
|
||||
branch = true
|
||||
|
|
@ -0,0 +1,128 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import trafilatura
|
||||
|
||||
from graphrag_platform.cli import stage_main
|
||||
from graphrag_platform.models import Document, RawDocument
|
||||
from graphrag_platform.runtime import StageContext
|
||||
from graphrag_platform.sources import SourceRecord, source_reader
|
||||
|
||||
|
||||
def content_hash(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def extract_document(record: SourceRecord) -> dict[str, object]:
|
||||
if "html" not in record.content_type.casefold():
|
||||
text = record.body.strip()
|
||||
if not text:
|
||||
raise ValueError(f"source content is empty for {record.url}")
|
||||
return {
|
||||
"title": record.metadata.get("title"),
|
||||
"author": record.metadata.get("author"),
|
||||
"date": record.metadata.get("published_at"),
|
||||
"hostname": None,
|
||||
"sitename": None,
|
||||
"text": text,
|
||||
}
|
||||
extracted = trafilatura.extract(
|
||||
record.body,
|
||||
url=str(record.url),
|
||||
output_format="json",
|
||||
with_metadata=True,
|
||||
include_comments=False,
|
||||
include_tables=True,
|
||||
favor_precision=True,
|
||||
)
|
||||
if not extracted:
|
||||
raise ValueError(f"could not extract article content from {record.url}")
|
||||
payload = json.loads(extracted)
|
||||
if not str(payload.get("text") or "").strip():
|
||||
raise ValueError(f"extracted content is empty for {record.url}")
|
||||
return payload
|
||||
|
||||
|
||||
def run(
|
||||
context: StageContext,
|
||||
*,
|
||||
input_jsonl: Path | None = None,
|
||||
) -> dict[str, int]:
|
||||
context.initialize()
|
||||
reader = source_reader(
|
||||
input_jsonl=input_jsonl,
|
||||
source_database_url=context.settings.source_database_url,
|
||||
source_table_name=context.settings.source_table_name,
|
||||
)
|
||||
normalized = 0
|
||||
with context.storage.session() as session:
|
||||
for record in reader.records():
|
||||
if record.corpus and record.corpus != context.settings.corpus_name:
|
||||
raise ValueError(
|
||||
f"record {record.external_id} targets corpus {record.corpus}, "
|
||||
f"not {context.settings.corpus_name}"
|
||||
)
|
||||
payload = extract_document(record)
|
||||
text = str(payload["text"]).strip()
|
||||
metadata = dict(record.metadata)
|
||||
metadata.update(
|
||||
{
|
||||
"external_id": record.external_id,
|
||||
"source_schema_version": record.schema_version,
|
||||
"retrieved_at": record.retrieved_at.isoformat(),
|
||||
"hostname": payload.get("hostname"),
|
||||
"sitename": payload.get("sitename"),
|
||||
}
|
||||
)
|
||||
raw = context.storage.upsert(
|
||||
session,
|
||||
RawDocument,
|
||||
{
|
||||
"corpus": context.settings.corpus_name,
|
||||
"external_id": record.external_id,
|
||||
},
|
||||
{
|
||||
"source_schema_version": record.schema_version,
|
||||
"url": str(record.url),
|
||||
"status_code": record.status_code,
|
||||
"content_type": record.content_type,
|
||||
"body": record.body,
|
||||
"metadata_json": record.metadata,
|
||||
"retrieved_at": record.retrieved_at,
|
||||
},
|
||||
)
|
||||
context.storage.upsert(
|
||||
session,
|
||||
Document,
|
||||
{"raw_document_id": raw.id},
|
||||
{
|
||||
"corpus": context.settings.corpus_name,
|
||||
"url": str(record.url),
|
||||
"title": payload.get("title"),
|
||||
"author": payload.get("author"),
|
||||
"published_at": payload.get("date"),
|
||||
"text": text,
|
||||
"content_hash": content_hash(text),
|
||||
"metadata_json": metadata,
|
||||
"is_duplicate": False,
|
||||
"canonical_document_id": None,
|
||||
},
|
||||
)
|
||||
normalized += 1
|
||||
if normalized == 0:
|
||||
raise ValueError("source handoff contained no records")
|
||||
return {"normalized": normalized}
|
||||
|
||||
|
||||
def run_from_args(
|
||||
context: StageContext, args: argparse.Namespace
|
||||
) -> dict[str, int]:
|
||||
return run(context, input_jsonl=args.input_jsonl)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stage_main(1, run_from_args)
|
||||
|
|
@ -0,0 +1,103 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import hashlib
|
||||
import re
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from graphrag_platform.cli import stage_main
|
||||
from graphrag_platform.models import Chunk, Document
|
||||
from graphrag_platform.runtime import StageContext
|
||||
|
||||
_SPACE_RE = re.compile(r"[ \t]+")
|
||||
_PARAGRAPH_RE = re.compile(r"\n{3,}")
|
||||
|
||||
|
||||
def content_hash(text: str) -> str:
|
||||
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def clean_text(text: str) -> str:
|
||||
lines = [_SPACE_RE.sub(" ", line).strip() for line in text.splitlines()]
|
||||
return _PARAGRAPH_RE.sub("\n\n", "\n".join(lines)).strip()
|
||||
|
||||
|
||||
def chunk_text(text: str, *, size: int, overlap: int) -> list[str]:
|
||||
if overlap >= size:
|
||||
raise ValueError("CHUNK_OVERLAP must be smaller than CHUNK_SIZE")
|
||||
chunks: list[str] = []
|
||||
start = 0
|
||||
while start < len(text):
|
||||
end = min(start + size, len(text))
|
||||
if end < len(text):
|
||||
break_at = text.rfind(" ", start + size // 2, end)
|
||||
if break_at > start:
|
||||
end = break_at
|
||||
chunk = text[start:end].strip()
|
||||
if chunk:
|
||||
chunks.append(chunk)
|
||||
if end >= len(text):
|
||||
break
|
||||
start = end - overlap
|
||||
return chunks
|
||||
|
||||
|
||||
def run(context: StageContext) -> dict[str, int]:
|
||||
context.initialize()
|
||||
chunk_count = 0
|
||||
duplicate_count = 0
|
||||
with context.storage.session() as session:
|
||||
documents = session.scalars(
|
||||
select(Document)
|
||||
.where(Document.corpus == context.settings.corpus_name)
|
||||
.order_by(Document.id)
|
||||
).all()
|
||||
if not documents:
|
||||
raise ValueError("stage 2 requires normalized documents")
|
||||
session.execute(
|
||||
delete(Chunk).where(Chunk.corpus == context.settings.corpus_name)
|
||||
)
|
||||
hashes: dict[str, int] = {}
|
||||
for document in documents:
|
||||
text = clean_text(document.text)
|
||||
digest = content_hash(text)
|
||||
document.text = text
|
||||
document.content_hash = digest
|
||||
if digest in hashes:
|
||||
document.is_duplicate = True
|
||||
document.canonical_document_id = hashes[digest]
|
||||
duplicate_count += 1
|
||||
continue
|
||||
document.is_duplicate = False
|
||||
document.canonical_document_id = None
|
||||
hashes[digest] = document.id
|
||||
for ordinal, text_chunk in enumerate(
|
||||
chunk_text(
|
||||
text,
|
||||
size=context.settings.chunk_size,
|
||||
overlap=context.settings.chunk_overlap,
|
||||
)
|
||||
):
|
||||
session.add(
|
||||
Chunk(
|
||||
document_id=document.id,
|
||||
corpus=context.settings.corpus_name,
|
||||
ordinal=ordinal,
|
||||
text=text_chunk,
|
||||
content_hash=content_hash(text_chunk),
|
||||
)
|
||||
)
|
||||
chunk_count += 1
|
||||
return {"chunks": chunk_count, "duplicates": duplicate_count}
|
||||
|
||||
|
||||
def run_from_args(
|
||||
context: StageContext, args: argparse.Namespace
|
||||
) -> dict[str, int]:
|
||||
del args
|
||||
return run(context)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stage_main(2, run_from_args)
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from graphrag_platform.cli import stage_main
|
||||
from graphrag_platform.models import KnowledgeModel
|
||||
from graphrag_platform.runtime import StageContext
|
||||
|
||||
|
||||
def validate_model_definition(definition: dict[str, Any]) -> None:
|
||||
for key in ("entity_types", "relationship_types"):
|
||||
value = definition.get(key)
|
||||
if not isinstance(value, list) or not value or not all(
|
||||
isinstance(item, str) and item.strip() for item in value
|
||||
):
|
||||
raise ValueError(f"{key} must be a non-empty array of strings")
|
||||
for key in ("node_template", "edge_template"):
|
||||
if not isinstance(definition.get(key), dict):
|
||||
raise ValueError(f"{key} must be a JSON object")
|
||||
|
||||
|
||||
def run(
|
||||
context: StageContext, *, model_file: Path | None = None
|
||||
) -> dict[str, int]:
|
||||
context.initialize()
|
||||
definition = context.settings.model_definition()
|
||||
if model_file is not None:
|
||||
loaded = json.loads(model_file.read_text(encoding="utf-8"))
|
||||
if not isinstance(loaded, dict):
|
||||
raise ValueError("knowledge model file must contain a JSON object")
|
||||
definition.update(loaded)
|
||||
validate_model_definition(definition)
|
||||
with context.storage.session() as session:
|
||||
context.storage.upsert(
|
||||
session,
|
||||
KnowledgeModel,
|
||||
{"corpus": context.settings.corpus_name},
|
||||
{"definition_json": definition},
|
||||
)
|
||||
return {
|
||||
"entity_types": len(definition["entity_types"]),
|
||||
"relationship_types": len(definition["relationship_types"]),
|
||||
}
|
||||
|
||||
|
||||
def run_from_args(
|
||||
context: StageContext, args: argparse.Namespace
|
||||
) -> dict[str, int]:
|
||||
return run(context, model_file=args.model_file)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stage_main(3, run_from_args)
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from graphrag_platform.cli import stage_main
|
||||
from graphrag_platform.models import Chunk, Extraction, KnowledgeModel
|
||||
from graphrag_platform.runtime import StageContext
|
||||
|
||||
|
||||
def validate_extraction(
|
||||
payload: dict[str, Any], definition: dict[str, Any]
|
||||
) -> None:
|
||||
entities = payload.get("entities")
|
||||
relationships = payload.get("relationships")
|
||||
if not isinstance(entities, list) or not isinstance(relationships, list):
|
||||
raise ValueError("extraction requires entities and relationships arrays")
|
||||
entity_types = set(definition["entity_types"])
|
||||
relationship_types = set(definition["relationship_types"])
|
||||
for entity in entities:
|
||||
if not isinstance(entity, dict):
|
||||
raise ValueError("each entity must be an object")
|
||||
if not str(entity.get("canonical_name", "")).strip():
|
||||
raise ValueError("entity canonical_name must not be empty")
|
||||
if entity.get("entity_type") not in entity_types:
|
||||
raise ValueError(f"unsupported entity type: {entity.get('entity_type')}")
|
||||
if not isinstance(entity.get("aliases", []), list):
|
||||
raise ValueError("entity aliases must be an array")
|
||||
for relationship in relationships:
|
||||
if not isinstance(relationship, dict):
|
||||
raise ValueError("each relationship must be an object")
|
||||
if not str(relationship.get("source", "")).strip() or not str(
|
||||
relationship.get("target", "")
|
||||
).strip():
|
||||
raise ValueError("relationship source and target must not be empty")
|
||||
if relationship.get("relationship_type") not in relationship_types:
|
||||
raise ValueError(
|
||||
"unsupported relationship type: "
|
||||
f"{relationship.get('relationship_type')}"
|
||||
)
|
||||
|
||||
|
||||
def run(context: StageContext) -> dict[str, int]:
|
||||
context.initialize()
|
||||
with context.storage.session() as session:
|
||||
model = session.get(KnowledgeModel, context.settings.corpus_name)
|
||||
chunks = session.scalars(
|
||||
select(Chunk).where(Chunk.corpus == context.settings.corpus_name)
|
||||
).all()
|
||||
if model is None or not chunks:
|
||||
raise ValueError("stage 4 requires a knowledge model and cleaned chunks")
|
||||
definition = dict(model.definition_json)
|
||||
chunk_inputs = [(chunk.id, chunk.text) for chunk in chunks]
|
||||
|
||||
system = (
|
||||
"You extract a source-grounded knowledge graph. Return one JSON object "
|
||||
'with arrays named "entities" and "relationships". Never infer unsupported facts.'
|
||||
)
|
||||
extracted_count = 0
|
||||
for chunk_id, chunk_text in chunk_inputs:
|
||||
prompt = json.dumps(
|
||||
{
|
||||
"knowledge_model": definition,
|
||||
"source_chunk_id": chunk_id,
|
||||
"text": chunk_text,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
)
|
||||
payload = context.llm.complete_json(system, prompt)
|
||||
validate_extraction(payload, definition)
|
||||
with context.storage.session() as session:
|
||||
context.storage.upsert(
|
||||
session,
|
||||
Extraction,
|
||||
{"chunk_id": chunk_id},
|
||||
{
|
||||
"corpus": context.settings.corpus_name,
|
||||
"payload_json": payload,
|
||||
"model_name": context.llm.chat_model,
|
||||
},
|
||||
)
|
||||
extracted_count += 1
|
||||
return {"extracted_chunks": extracted_count}
|
||||
|
||||
|
||||
def run_from_args(
|
||||
context: StageContext, args: argparse.Namespace
|
||||
) -> dict[str, int]:
|
||||
del args
|
||||
return run(context)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stage_main(4, run_from_args)
|
||||
|
|
@ -0,0 +1,137 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import re
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from graphrag_platform.cli import stage_main
|
||||
from graphrag_platform.models import (
|
||||
Extraction,
|
||||
ResolvedEntity,
|
||||
ResolvedRelationship,
|
||||
)
|
||||
from graphrag_platform.runtime import StageContext
|
||||
|
||||
_KEY_RE = re.compile(r"[^\w]+", re.UNICODE)
|
||||
|
||||
|
||||
def canonical_key(name: str) -> str:
|
||||
return _KEY_RE.sub(" ", name.casefold()).strip()
|
||||
|
||||
|
||||
def merge_text(current: str, incoming: str) -> str:
|
||||
current = current.strip()
|
||||
incoming = incoming.strip()
|
||||
if not incoming or incoming.casefold() in current.casefold():
|
||||
return current
|
||||
if not current:
|
||||
return incoming
|
||||
return f"{current}\n{incoming}"
|
||||
|
||||
|
||||
def run(context: StageContext) -> dict[str, int]:
|
||||
context.initialize()
|
||||
entity_count = 0
|
||||
relationship_count = 0
|
||||
with context.storage.session() as session:
|
||||
extractions = session.scalars(
|
||||
select(Extraction).where(
|
||||
Extraction.corpus == context.settings.corpus_name
|
||||
)
|
||||
).all()
|
||||
if not extractions:
|
||||
raise ValueError("stage 5 requires extracted knowledge")
|
||||
session.execute(
|
||||
delete(ResolvedRelationship).where(
|
||||
ResolvedRelationship.corpus == context.settings.corpus_name
|
||||
)
|
||||
)
|
||||
session.execute(
|
||||
delete(ResolvedEntity).where(
|
||||
ResolvedEntity.corpus == context.settings.corpus_name
|
||||
)
|
||||
)
|
||||
entity_rows: dict[str, dict[str, Any]] = {}
|
||||
relation_rows: dict[tuple[str, str, str], dict[str, Any]] = {}
|
||||
pending_relationships: list[tuple[int, dict[str, Any]]] = []
|
||||
for extraction in extractions:
|
||||
payload = extraction.payload_json
|
||||
for entity in payload["entities"]:
|
||||
name = entity["canonical_name"].strip()
|
||||
key = canonical_key(name)
|
||||
row = entity_rows.get(key)
|
||||
if row is None:
|
||||
row = {
|
||||
"canonical_name": name,
|
||||
"entity_type": entity["entity_type"],
|
||||
"description": "",
|
||||
"aliases": set(),
|
||||
"chunk_ids": set(),
|
||||
}
|
||||
entity_rows[key] = row
|
||||
elif row["entity_type"] != entity["entity_type"]:
|
||||
raise ValueError(
|
||||
f"conflicting entity types for {name}: "
|
||||
f"{row['entity_type']} and {entity['entity_type']}"
|
||||
)
|
||||
row["description"] = merge_text(
|
||||
row["description"], entity.get("description", "")
|
||||
)
|
||||
row["aliases"].update(entity.get("aliases", []))
|
||||
row["aliases"].add(name)
|
||||
row["chunk_ids"].add(extraction.chunk_id)
|
||||
for relationship in payload["relationships"]:
|
||||
pending_relationships.append((extraction.chunk_id, relationship))
|
||||
for chunk_id, relationship in pending_relationships:
|
||||
source_key = canonical_key(relationship["source"])
|
||||
target_key = canonical_key(relationship["target"])
|
||||
if source_key not in entity_rows or target_key not in entity_rows:
|
||||
continue
|
||||
relation_type = relationship["relationship_type"]
|
||||
identity = (source_key, target_key, relation_type)
|
||||
row = relation_rows.setdefault(
|
||||
identity, {"description": "", "chunk_ids": set()}
|
||||
)
|
||||
row["description"] = merge_text(
|
||||
row["description"], relationship.get("description", "")
|
||||
)
|
||||
row["chunk_ids"].add(chunk_id)
|
||||
for key, row in entity_rows.items():
|
||||
session.add(
|
||||
ResolvedEntity(
|
||||
corpus=context.settings.corpus_name,
|
||||
canonical_key=key,
|
||||
canonical_name=row["canonical_name"],
|
||||
entity_type=row["entity_type"],
|
||||
description=row["description"],
|
||||
aliases_json=sorted(row["aliases"], key=str.casefold),
|
||||
source_chunk_ids_json=sorted(row["chunk_ids"]),
|
||||
)
|
||||
)
|
||||
entity_count += 1
|
||||
for (source_key, target_key, relation_type), row in relation_rows.items():
|
||||
session.add(
|
||||
ResolvedRelationship(
|
||||
corpus=context.settings.corpus_name,
|
||||
source_key=source_key,
|
||||
target_key=target_key,
|
||||
relationship_type=relation_type,
|
||||
description=row["description"],
|
||||
source_chunk_ids_json=sorted(row["chunk_ids"]),
|
||||
)
|
||||
)
|
||||
relationship_count += 1
|
||||
return {"entities": entity_count, "relationships": relationship_count}
|
||||
|
||||
|
||||
def run_from_args(
|
||||
context: StageContext, args: argparse.Namespace
|
||||
) -> dict[str, int]:
|
||||
del args
|
||||
return run(context)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stage_main(5, run_from_args)
|
||||
|
|
@ -0,0 +1,80 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from graphrag_platform.cli import stage_main
|
||||
from graphrag_platform.models import (
|
||||
Edge,
|
||||
Node,
|
||||
ResolvedEntity,
|
||||
ResolvedRelationship,
|
||||
)
|
||||
from graphrag_platform.runtime import StageContext
|
||||
|
||||
|
||||
def run(context: StageContext) -> dict[str, int]:
|
||||
context.initialize()
|
||||
with context.storage.session() as session:
|
||||
entities = session.scalars(
|
||||
select(ResolvedEntity).where(
|
||||
ResolvedEntity.corpus == context.settings.corpus_name
|
||||
)
|
||||
).all()
|
||||
relationships = session.scalars(
|
||||
select(ResolvedRelationship).where(
|
||||
ResolvedRelationship.corpus == context.settings.corpus_name
|
||||
)
|
||||
).all()
|
||||
if not entities:
|
||||
raise ValueError("stage 6 requires reconciled entities")
|
||||
session.execute(
|
||||
delete(Edge).where(Edge.corpus == context.settings.corpus_name)
|
||||
)
|
||||
session.execute(
|
||||
delete(Node).where(Node.corpus == context.settings.corpus_name)
|
||||
)
|
||||
node_ids: dict[str, int] = {}
|
||||
for entity in entities:
|
||||
node = Node(
|
||||
corpus=entity.corpus,
|
||||
canonical_key=entity.canonical_key,
|
||||
canonical_name=entity.canonical_name,
|
||||
entity_type=entity.entity_type,
|
||||
description=entity.description,
|
||||
aliases_json=entity.aliases_json,
|
||||
source_chunk_ids_json=entity.source_chunk_ids_json,
|
||||
)
|
||||
session.add(node)
|
||||
session.flush()
|
||||
node_ids[entity.canonical_key] = node.id
|
||||
edge_count = 0
|
||||
for relationship in relationships:
|
||||
source_id = node_ids.get(relationship.source_key)
|
||||
target_id = node_ids.get(relationship.target_key)
|
||||
if source_id is None or target_id is None:
|
||||
continue
|
||||
session.add(
|
||||
Edge(
|
||||
corpus=relationship.corpus,
|
||||
source_node_id=source_id,
|
||||
target_node_id=target_id,
|
||||
relationship_type=relationship.relationship_type,
|
||||
description=relationship.description,
|
||||
source_chunk_ids_json=relationship.source_chunk_ids_json,
|
||||
)
|
||||
)
|
||||
edge_count += 1
|
||||
return {"nodes": len(node_ids), "edges": edge_count}
|
||||
|
||||
|
||||
def run_from_args(
|
||||
context: StageContext, args: argparse.Namespace
|
||||
) -> dict[str, int]:
|
||||
del args
|
||||
return run(context)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stage_main(6, run_from_args)
|
||||
|
|
@ -0,0 +1,97 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
|
||||
import networkx as nx
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from graphrag_platform.cli import stage_main
|
||||
from graphrag_platform.models import Community, Edge, Node
|
||||
from graphrag_platform.runtime import StageContext
|
||||
|
||||
|
||||
def run(context: StageContext) -> dict[str, int]:
|
||||
context.initialize()
|
||||
with context.storage.session() as session:
|
||||
nodes = session.scalars(
|
||||
select(Node).where(Node.corpus == context.settings.corpus_name)
|
||||
).all()
|
||||
edges = session.scalars(
|
||||
select(Edge).where(Edge.corpus == context.settings.corpus_name)
|
||||
).all()
|
||||
if not nodes:
|
||||
raise ValueError("stage 7 requires graph nodes")
|
||||
graph = nx.Graph()
|
||||
graph.add_nodes_from(node.id for node in nodes)
|
||||
graph.add_edges_from(
|
||||
(edge.source_node_id, edge.target_node_id) for edge in edges
|
||||
)
|
||||
if graph.number_of_edges():
|
||||
groups = list(
|
||||
nx.community.greedy_modularity_communities(
|
||||
graph, resolution=context.settings.community_resolution
|
||||
)
|
||||
)
|
||||
else:
|
||||
groups = [{node.id} for node in nodes]
|
||||
by_id = {node.id: node for node in nodes}
|
||||
community_inputs = [
|
||||
(
|
||||
index,
|
||||
sorted(group),
|
||||
[
|
||||
{
|
||||
"name": by_id[node_id].canonical_name,
|
||||
"type": by_id[node_id].entity_type,
|
||||
"description": by_id[node_id].description,
|
||||
}
|
||||
for node_id in sorted(group)
|
||||
],
|
||||
)
|
||||
for index, group in enumerate(groups, start=1)
|
||||
]
|
||||
|
||||
summaries: list[tuple[int, list[int], str, str]] = []
|
||||
for index, node_ids, evidence in community_inputs:
|
||||
summary = context.llm.complete_json(
|
||||
(
|
||||
"Summarize a knowledge-graph community using only the supplied "
|
||||
'evidence. Return JSON with non-empty "title" and "summary" strings.'
|
||||
),
|
||||
json.dumps(evidence, ensure_ascii=False),
|
||||
)
|
||||
title = str(summary.get("title", "")).strip()
|
||||
description = str(summary.get("summary", "")).strip()
|
||||
if not title or not description:
|
||||
raise ValueError("community summary requires title and summary")
|
||||
summaries.append((index, node_ids, title, description))
|
||||
|
||||
with context.storage.session() as session:
|
||||
session.execute(
|
||||
delete(Community).where(
|
||||
Community.corpus == context.settings.corpus_name
|
||||
)
|
||||
)
|
||||
for index, node_ids, title, description in summaries:
|
||||
session.add(
|
||||
Community(
|
||||
corpus=context.settings.corpus_name,
|
||||
community_key=f"community-{index}",
|
||||
title=title,
|
||||
summary=description,
|
||||
node_ids_json=node_ids,
|
||||
)
|
||||
)
|
||||
return {"communities": len(groups)}
|
||||
|
||||
|
||||
def run_from_args(
|
||||
context: StageContext, args: argparse.Namespace
|
||||
) -> dict[str, int]:
|
||||
del args
|
||||
return run(context)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stage_main(7, run_from_args)
|
||||
|
|
@ -0,0 +1,164 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from graphrag_platform.cli import stage_main
|
||||
from graphrag_platform.models import (
|
||||
Chunk,
|
||||
Community,
|
||||
Document,
|
||||
Edge,
|
||||
Node,
|
||||
SearchItem,
|
||||
)
|
||||
from graphrag_platform.runtime import StageContext
|
||||
|
||||
|
||||
def run(context: StageContext) -> dict[str, int]:
|
||||
context.initialize()
|
||||
with context.storage.session() as session:
|
||||
chunks = session.scalars(
|
||||
select(Chunk).where(Chunk.corpus == context.settings.corpus_name)
|
||||
).all()
|
||||
nodes = session.scalars(
|
||||
select(Node).where(Node.corpus == context.settings.corpus_name)
|
||||
).all()
|
||||
edges = session.scalars(
|
||||
select(Edge).where(Edge.corpus == context.settings.corpus_name)
|
||||
).all()
|
||||
communities = session.scalars(
|
||||
select(Community).where(
|
||||
Community.corpus == context.settings.corpus_name
|
||||
)
|
||||
).all()
|
||||
if not nodes or not chunks:
|
||||
raise ValueError("stage 8 requires graph nodes and source chunks")
|
||||
documents = {
|
||||
document.id: document
|
||||
for document in session.scalars(
|
||||
select(Document).where(
|
||||
Document.corpus == context.settings.corpus_name
|
||||
)
|
||||
).all()
|
||||
}
|
||||
node_names = {node.id: node.canonical_name for node in nodes}
|
||||
node_source_chunks = {
|
||||
node.id: set(node.source_chunk_ids_json) for node in nodes
|
||||
}
|
||||
candidates: list[dict[str, Any]] = []
|
||||
for chunk in chunks:
|
||||
document = documents[chunk.document_id]
|
||||
candidates.append(
|
||||
{
|
||||
"item_type": "chunk",
|
||||
"item_key": str(chunk.id),
|
||||
"text": chunk.text,
|
||||
"source_url": document.url,
|
||||
"metadata": {
|
||||
"document_id": document.id,
|
||||
"title": document.title,
|
||||
"chunk_ordinal": chunk.ordinal,
|
||||
},
|
||||
}
|
||||
)
|
||||
for node in nodes:
|
||||
candidates.append(
|
||||
{
|
||||
"item_type": "node",
|
||||
"item_key": str(node.id),
|
||||
"text": (
|
||||
f"{node.canonical_name} ({node.entity_type}): "
|
||||
f"{node.description}"
|
||||
),
|
||||
"source_url": None,
|
||||
"metadata": {"source_chunk_ids": node.source_chunk_ids_json},
|
||||
}
|
||||
)
|
||||
for edge in edges:
|
||||
candidates.append(
|
||||
{
|
||||
"item_type": "edge",
|
||||
"item_key": str(edge.id),
|
||||
"text": (
|
||||
f"{node_names[edge.source_node_id]} "
|
||||
f"{edge.relationship_type} "
|
||||
f"{node_names[edge.target_node_id]}: {edge.description}"
|
||||
),
|
||||
"source_url": None,
|
||||
"metadata": {
|
||||
"source_node_id": edge.source_node_id,
|
||||
"target_node_id": edge.target_node_id,
|
||||
"relationship_type": edge.relationship_type,
|
||||
"source_chunk_ids": edge.source_chunk_ids_json,
|
||||
},
|
||||
}
|
||||
)
|
||||
for community in communities:
|
||||
community_chunk_ids: set[int] = set()
|
||||
for node_id in community.node_ids_json:
|
||||
community_chunk_ids.update(node_source_chunks.get(node_id, set()))
|
||||
candidates.append(
|
||||
{
|
||||
"item_type": "community",
|
||||
"item_key": str(community.id),
|
||||
"text": f"{community.title}: {community.summary}",
|
||||
"source_url": None,
|
||||
"metadata": {
|
||||
"node_ids": community.node_ids_json,
|
||||
"source_chunk_ids": sorted(community_chunk_ids),
|
||||
},
|
||||
}
|
||||
)
|
||||
embeddings: list[list[float]] = []
|
||||
for offset in range(0, len(candidates), 64):
|
||||
embeddings.extend(
|
||||
context.llm.embed(
|
||||
[row["text"] for row in candidates[offset : offset + 64]]
|
||||
)
|
||||
)
|
||||
if len(embeddings) != len(candidates):
|
||||
raise ValueError("embedding provider returned an unexpected row count")
|
||||
dimensions = context.settings.llm_embedding_dimensions
|
||||
invalid_dimensions = [
|
||||
len(embedding) for embedding in embeddings if len(embedding) != dimensions
|
||||
]
|
||||
if invalid_dimensions:
|
||||
raise ValueError(
|
||||
"embedding provider returned vectors with unexpected dimensions: "
|
||||
f"expected {dimensions}, received {invalid_dimensions[0]}"
|
||||
)
|
||||
with context.storage.session() as session:
|
||||
session.execute(
|
||||
delete(SearchItem).where(
|
||||
SearchItem.corpus == context.settings.corpus_name
|
||||
)
|
||||
)
|
||||
for candidate, embedding in zip(candidates, embeddings, strict=True):
|
||||
session.add(
|
||||
SearchItem(
|
||||
corpus=context.settings.corpus_name,
|
||||
item_type=candidate["item_type"],
|
||||
item_key=candidate["item_key"],
|
||||
text=candidate["text"],
|
||||
source_url=candidate["source_url"],
|
||||
metadata_json=candidate["metadata"],
|
||||
embedding=embedding,
|
||||
embedding_model=context.llm.embedding_model,
|
||||
)
|
||||
)
|
||||
context.storage.ensure_vector_index(context.settings.llm_embedding_dimensions)
|
||||
return {"indexed_items": len(candidates)}
|
||||
|
||||
|
||||
def run_from_args(
|
||||
context: StageContext, args: argparse.Namespace
|
||||
) -> dict[str, int]:
|
||||
del args
|
||||
return run(context)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stage_main(8, run_from_args)
|
||||
|
|
@ -0,0 +1,238 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import math
|
||||
from typing import Any
|
||||
|
||||
from pgvector.sqlalchemy import Vector
|
||||
from sqlalchemy import func, or_, select, text
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from graphrag_platform.cli import stage_main
|
||||
from graphrag_platform.models import Chunk, Document, Edge, Node, SearchItem
|
||||
from graphrag_platform.runtime import StageContext
|
||||
|
||||
|
||||
def cosine_similarity(left: list[float], right: list[float]) -> float:
|
||||
if len(left) != len(right) or not left:
|
||||
return 0.0
|
||||
denominator = math.sqrt(sum(x * x for x in left)) * math.sqrt(
|
||||
sum(x * x for x in right)
|
||||
)
|
||||
if denominator == 0:
|
||||
return 0.0
|
||||
return sum(x * y for x, y in zip(left, right, strict=True)) / denominator
|
||||
|
||||
|
||||
def sources_for_chunks(
|
||||
session: Session, chunk_ids: set[int]
|
||||
) -> list[dict[str, Any]]:
|
||||
if not chunk_ids:
|
||||
return []
|
||||
rows = session.execute(
|
||||
select(Chunk, Document)
|
||||
.join(Document, Chunk.document_id == Document.id)
|
||||
.where(Chunk.id.in_(chunk_ids))
|
||||
.order_by(Document.url, Chunk.ordinal)
|
||||
).all()
|
||||
return [
|
||||
{
|
||||
"citation": f"S{index}",
|
||||
"url": document.url,
|
||||
"title": document.title,
|
||||
"chunk_id": chunk.id,
|
||||
"text": chunk.text,
|
||||
}
|
||||
for index, (chunk, document) in enumerate(rows, start=1)
|
||||
]
|
||||
|
||||
|
||||
def rank_vector_context(
|
||||
context: StageContext,
|
||||
session: Session,
|
||||
query_embedding: list[float],
|
||||
limit: int,
|
||||
) -> list[tuple[float, SearchItem]]:
|
||||
if context.storage.supports_vector_search:
|
||||
dimensions = context.settings.llm_embedding_dimensions
|
||||
session.execute(
|
||||
text(
|
||||
"SET LOCAL hnsw.ef_search = "
|
||||
f"{context.settings.retrieval_hnsw_ef_search}"
|
||||
)
|
||||
)
|
||||
session.execute(text("SET LOCAL hnsw.iterative_scan = strict_order"))
|
||||
distance = (
|
||||
SearchItem.embedding.cast(Vector(dimensions))
|
||||
.cosine_distance(query_embedding)
|
||||
.label("distance")
|
||||
)
|
||||
rows = session.execute(
|
||||
select(SearchItem, distance)
|
||||
.where(
|
||||
SearchItem.corpus == context.settings.corpus_name,
|
||||
SearchItem.embedding_model == context.llm.embedding_model,
|
||||
func.vector_dims(SearchItem.embedding) == dimensions,
|
||||
)
|
||||
.order_by(distance)
|
||||
.limit(limit)
|
||||
).all()
|
||||
return [(1.0 - float(distance_value), item) for item, distance_value in rows]
|
||||
|
||||
search_items = session.scalars(
|
||||
select(SearchItem).where(
|
||||
SearchItem.corpus == context.settings.corpus_name,
|
||||
SearchItem.embedding_model == context.llm.embedding_model,
|
||||
)
|
||||
).all()
|
||||
return sorted(
|
||||
(
|
||||
(cosine_similarity(query_embedding, list(item.embedding)), item)
|
||||
for item in search_items
|
||||
),
|
||||
key=lambda pair: pair[0],
|
||||
reverse=True,
|
||||
)[:limit]
|
||||
|
||||
|
||||
def expand_graph(
|
||||
context: StageContext,
|
||||
session: Session,
|
||||
ranked: list[tuple[float, SearchItem]],
|
||||
) -> tuple[list[dict[str, Any]], set[int]]:
|
||||
seed_node_ids: set[int] = set()
|
||||
source_chunk_ids: set[int] = set()
|
||||
for _, item in ranked:
|
||||
source_chunk_ids.update(item.metadata_json.get("source_chunk_ids", []))
|
||||
if item.item_type == "node":
|
||||
seed_node_ids.add(int(item.item_key))
|
||||
elif item.item_type == "edge":
|
||||
seed_node_ids.update(
|
||||
{
|
||||
int(item.metadata_json["source_node_id"]),
|
||||
int(item.metadata_json["target_node_id"]),
|
||||
}
|
||||
)
|
||||
elif item.item_type == "community":
|
||||
seed_node_ids.update(item.metadata_json.get("node_ids", []))
|
||||
|
||||
if not seed_node_ids or context.settings.retrieval_graph_hops == 0:
|
||||
return [], source_chunk_ids
|
||||
|
||||
visited_nodes = set(seed_node_ids)
|
||||
visited_edges: dict[int, Edge] = {}
|
||||
frontier = set(seed_node_ids)
|
||||
for _ in range(context.settings.retrieval_graph_hops):
|
||||
if not frontier or len(visited_edges) >= context.settings.retrieval_graph_limit:
|
||||
break
|
||||
remaining = context.settings.retrieval_graph_limit - len(visited_edges)
|
||||
edges = session.scalars(
|
||||
select(Edge)
|
||||
.where(
|
||||
Edge.corpus == context.settings.corpus_name,
|
||||
or_(
|
||||
Edge.source_node_id.in_(frontier),
|
||||
Edge.target_node_id.in_(frontier),
|
||||
),
|
||||
)
|
||||
.order_by(Edge.weight.desc(), Edge.id)
|
||||
.limit(remaining)
|
||||
).all()
|
||||
next_frontier: set[int] = set()
|
||||
for edge in edges:
|
||||
visited_edges[edge.id] = edge
|
||||
source_chunk_ids.update(edge.source_chunk_ids_json)
|
||||
next_frontier.update({edge.source_node_id, edge.target_node_id})
|
||||
next_frontier -= visited_nodes
|
||||
visited_nodes.update(next_frontier)
|
||||
frontier = next_frontier
|
||||
|
||||
nodes = {
|
||||
node.id: node
|
||||
for node in session.scalars(
|
||||
select(Node).where(
|
||||
Node.corpus == context.settings.corpus_name,
|
||||
Node.id.in_(visited_nodes),
|
||||
)
|
||||
).all()
|
||||
}
|
||||
for node in nodes.values():
|
||||
source_chunk_ids.update(node.source_chunk_ids_json)
|
||||
graph_context = [
|
||||
{
|
||||
"edge_id": edge.id,
|
||||
"source_node_id": edge.source_node_id,
|
||||
"source": nodes[edge.source_node_id].canonical_name,
|
||||
"relationship": edge.relationship_type,
|
||||
"target_node_id": edge.target_node_id,
|
||||
"target": nodes[edge.target_node_id].canonical_name,
|
||||
"description": edge.description,
|
||||
"weight": edge.weight,
|
||||
"source_chunk_ids": edge.source_chunk_ids_json,
|
||||
}
|
||||
for edge in visited_edges.values()
|
||||
if edge.source_node_id in nodes and edge.target_node_id in nodes
|
||||
]
|
||||
return graph_context, source_chunk_ids
|
||||
|
||||
|
||||
def run(
|
||||
context: StageContext,
|
||||
*,
|
||||
question: str,
|
||||
top_k: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
context.initialize()
|
||||
if not question.strip():
|
||||
raise ValueError("question must not be empty")
|
||||
query_embedding = context.llm.embed([question])[0]
|
||||
expected_dimensions = context.settings.llm_embedding_dimensions
|
||||
if len(query_embedding) != expected_dimensions:
|
||||
raise ValueError(
|
||||
"embedding provider returned a query vector with unexpected dimensions: "
|
||||
f"expected {expected_dimensions}, received {len(query_embedding)}"
|
||||
)
|
||||
with context.storage.session() as session:
|
||||
limit = top_k or context.settings.retrieval_top_k
|
||||
ranked = rank_vector_context(
|
||||
context,
|
||||
session,
|
||||
query_embedding,
|
||||
limit,
|
||||
)
|
||||
if not ranked:
|
||||
raise ValueError("stage 9 requires a retrieval index")
|
||||
graph_context, chunk_ids = expand_graph(context, session, ranked)
|
||||
for _, item in ranked:
|
||||
if item.item_type == "chunk":
|
||||
chunk_ids.add(int(item.item_key))
|
||||
chunk_ids.update(item.metadata_json.get("source_chunk_ids", []))
|
||||
sources = sources_for_chunks(session, chunk_ids)
|
||||
return {
|
||||
"question": question,
|
||||
"context": [
|
||||
{
|
||||
"score": round(score, 6),
|
||||
"type": item.item_type,
|
||||
"key": item.item_key,
|
||||
"text": item.text,
|
||||
"source_url": item.source_url,
|
||||
"metadata": item.metadata_json,
|
||||
}
|
||||
for score, item in ranked
|
||||
],
|
||||
"graph_context": graph_context,
|
||||
"sources": sources,
|
||||
}
|
||||
|
||||
|
||||
def run_from_args(
|
||||
context: StageContext, args: argparse.Namespace
|
||||
) -> dict[str, Any]:
|
||||
if not args.question:
|
||||
raise ValueError("--question is required for stage 9")
|
||||
return run(context, question=args.question, top_k=args.top_k)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stage_main(9, run_from_args)
|
||||
|
|
@ -0,0 +1,47 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from graphrag_platform.cli import stage_main
|
||||
from graphrag_platform.runtime import StageContext
|
||||
from graphrag_platform.stage_loader import load_stage
|
||||
|
||||
|
||||
def run(
|
||||
context: StageContext,
|
||||
*,
|
||||
question: str,
|
||||
top_k: int | None = None,
|
||||
) -> dict[str, Any]:
|
||||
retrieval = load_stage(9).run(
|
||||
context,
|
||||
question=question,
|
||||
top_k=top_k,
|
||||
)
|
||||
answer = context.llm.complete_text(
|
||||
(
|
||||
"Answer only from the supplied GraphRAG context. Cite supporting sources "
|
||||
"using [S1], [S2], and so on. If the context does not support an answer, "
|
||||
"state that clearly. Do not invent citations."
|
||||
),
|
||||
json.dumps(retrieval, ensure_ascii=False),
|
||||
)
|
||||
return {
|
||||
"question": question,
|
||||
"answer": answer,
|
||||
"sources": retrieval["sources"],
|
||||
}
|
||||
|
||||
|
||||
def run_from_args(
|
||||
context: StageContext, args: argparse.Namespace
|
||||
) -> dict[str, Any]:
|
||||
if not args.question:
|
||||
raise ValueError("--question is required for stage 10")
|
||||
return run(context, question=args.question, top_k=args.top_k)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
stage_main(10, run_from_args)
|
||||
|
|
@ -0,0 +1,6 @@
|
|||
"""KnowledgeGraph GraphRAG platform."""
|
||||
|
||||
from graphrag_platform.config import Settings
|
||||
from graphrag_platform.pipeline import KnowledgeGraphPipeline
|
||||
|
||||
__all__ = ["KnowledgeGraphPipeline", "Settings"]
|
||||
|
|
@ -0,0 +1,247 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from typing import Annotated, Any, Callable
|
||||
|
||||
import jwt
|
||||
from fastapi import Depends, FastAPI, HTTPException, Request, status
|
||||
from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer
|
||||
from pydantic import BaseModel, Field
|
||||
from sqlalchemy import select, text
|
||||
|
||||
from graphrag_platform.auth import Permission, Principal, decode_internal_token
|
||||
from graphrag_platform.config import Settings
|
||||
from graphrag_platform.llm import LLMClient
|
||||
from graphrag_platform.models import PipelineRun, StageRun
|
||||
from graphrag_platform.observability import configure_logging
|
||||
from graphrag_platform.orchestrator import requeue_failed_run, submit_run
|
||||
from graphrag_platform.runtime import StageContext
|
||||
from graphrag_platform.stage_loader import load_stage
|
||||
from graphrag_platform.storage import Storage
|
||||
|
||||
bearer = HTTPBearer(auto_error=False)
|
||||
|
||||
|
||||
class JobRequest(BaseModel):
|
||||
corpus: str = Field(min_length=1, max_length=200)
|
||||
from_stage: int = Field(default=1, ge=1, le=8)
|
||||
through_stage: int = Field(default=8, ge=1, le=8)
|
||||
input_jsonl: str | None = None
|
||||
model_file: str | None = None
|
||||
reset: bool = False
|
||||
|
||||
|
||||
class QueryRequest(BaseModel):
|
||||
corpus: str = Field(min_length=1, max_length=200)
|
||||
question: str = Field(min_length=1)
|
||||
top_k: int | None = Field(default=None, ge=1)
|
||||
|
||||
|
||||
def _principal(
|
||||
request: Request,
|
||||
credentials: Annotated[
|
||||
HTTPAuthorizationCredentials | None, Depends(bearer)
|
||||
],
|
||||
) -> Principal:
|
||||
if credentials is None:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Bearer service token is required",
|
||||
)
|
||||
try:
|
||||
return decode_internal_token(credentials.credentials, request.app.state.settings)
|
||||
except jwt.PyJWTError as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid service token",
|
||||
) from error
|
||||
|
||||
|
||||
def require_permissions(
|
||||
required: Permission,
|
||||
) -> Callable[[Principal], Principal]:
|
||||
def dependency(principal: Annotated[Principal, Depends(_principal)]) -> Principal:
|
||||
if not principal.has(required):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail=f"KnowledgeGraph permission mask {int(required)} is required",
|
||||
)
|
||||
return principal
|
||||
|
||||
return dependency
|
||||
|
||||
|
||||
def create_app(
|
||||
settings: Settings | None = None,
|
||||
*,
|
||||
llm: LLMClient | None = None,
|
||||
) -> FastAPI:
|
||||
active_settings = settings or Settings()
|
||||
configure_logging(active_settings.log_level)
|
||||
app = FastAPI(
|
||||
title="KnowledgeGraph Internal API",
|
||||
version="1.0.0",
|
||||
docs_url=None,
|
||||
redoc_url=None,
|
||||
openapi_url="/internal/knowledge-graph/openapi.json",
|
||||
)
|
||||
app.state.settings = active_settings
|
||||
app.state.storage = Storage(active_settings.database_url)
|
||||
app.state.llm = llm
|
||||
|
||||
def context(corpus: str) -> StageContext:
|
||||
corpus_settings = active_settings.model_copy(update={"corpus_name": corpus})
|
||||
return StageContext.create(
|
||||
corpus_settings,
|
||||
storage=app.state.storage,
|
||||
llm=app.state.llm,
|
||||
)
|
||||
|
||||
@app.get("/internal/knowledge-graph/health")
|
||||
def health() -> dict[str, str]:
|
||||
return {"status": "healthy"}
|
||||
|
||||
@app.get("/internal/knowledge-graph/ready")
|
||||
def ready() -> dict[str, str]:
|
||||
try:
|
||||
with app.state.storage.engine.connect() as connection:
|
||||
connection.execute(text("SELECT 1"))
|
||||
if connection.dialect.name == "postgresql":
|
||||
extension = connection.scalar(
|
||||
text(
|
||||
"SELECT extname FROM pg_extension "
|
||||
"WHERE extname = 'vector'"
|
||||
)
|
||||
)
|
||||
if extension != "vector":
|
||||
raise RuntimeError("PgVector extension is not installed")
|
||||
except Exception as error:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_503_SERVICE_UNAVAILABLE,
|
||||
detail="Service dependencies are not ready",
|
||||
) from error
|
||||
return {"status": "ready"}
|
||||
|
||||
@app.post(
|
||||
"/internal/knowledge-graph/jobs",
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
)
|
||||
def create_job(
|
||||
body: JobRequest,
|
||||
principal: Annotated[
|
||||
Principal, Depends(require_permissions(Permission.EXECUTE))
|
||||
],
|
||||
) -> dict[str, str]:
|
||||
if body.from_stage == 1 and not principal.has(Permission.ADD):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_403_FORBIDDEN,
|
||||
detail="KnowledgeGraph Add permission is required for stage 1",
|
||||
)
|
||||
if body.from_stage > body.through_stage:
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
detail="from_stage must not exceed through_stage",
|
||||
)
|
||||
run_context = context(body.corpus)
|
||||
run_context.initialize()
|
||||
run_id = submit_run(
|
||||
run_context,
|
||||
from_stage=body.from_stage,
|
||||
through_stage=body.through_stage,
|
||||
payload={
|
||||
"input_jsonl": body.input_jsonl,
|
||||
"model_file": body.model_file,
|
||||
"reset": body.reset,
|
||||
},
|
||||
requested_by=principal.subject,
|
||||
correlation_id=principal.correlation_id,
|
||||
permission_mask=principal.permission_mask,
|
||||
)
|
||||
return {"job_id": run_id, "status": "queued"}
|
||||
|
||||
@app.get("/internal/knowledge-graph/jobs/{job_id}")
|
||||
def get_job(
|
||||
job_id: str,
|
||||
principal: Annotated[
|
||||
Principal, Depends(require_permissions(Permission.READ))
|
||||
],
|
||||
) -> dict[str, Any]:
|
||||
del principal
|
||||
with app.state.storage.session() as session:
|
||||
run = session.get(PipelineRun, job_id)
|
||||
if run is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
stages = session.scalars(
|
||||
select(StageRun)
|
||||
.where(StageRun.pipeline_run_id == job_id)
|
||||
.order_by(StageRun.stage_number, StageRun.attempt)
|
||||
).all()
|
||||
return {
|
||||
"job_id": run.id,
|
||||
"corpus": run.corpus,
|
||||
"status": run.status,
|
||||
"attempt": run.attempt,
|
||||
"from_stage": run.from_stage,
|
||||
"through_stage": run.through_stage,
|
||||
"error": run.error_message,
|
||||
"output": run.output_json,
|
||||
"stages": [
|
||||
{
|
||||
"stage": stage_run.stage_number,
|
||||
"name": stage_run.stage_name,
|
||||
"status": stage_run.status,
|
||||
"attempt": stage_run.attempt,
|
||||
"output": stage_run.output_json,
|
||||
"error": stage_run.error_message,
|
||||
}
|
||||
for stage_run in stages
|
||||
],
|
||||
}
|
||||
|
||||
@app.post(
|
||||
"/internal/knowledge-graph/jobs/{job_id}/retry",
|
||||
status_code=status.HTTP_202_ACCEPTED,
|
||||
)
|
||||
def retry_job(
|
||||
job_id: str,
|
||||
principal: Annotated[
|
||||
Principal, Depends(require_permissions(Permission.EXECUTE))
|
||||
],
|
||||
) -> dict[str, str]:
|
||||
del principal
|
||||
requeue_failed_run(context(active_settings.corpus_name), job_id)
|
||||
return {"job_id": job_id, "status": "queued"}
|
||||
|
||||
@app.post("/internal/knowledge-graph/query")
|
||||
def query(
|
||||
body: QueryRequest,
|
||||
principal: Annotated[
|
||||
Principal,
|
||||
Depends(require_permissions(Permission.READ | Permission.EXECUTE)),
|
||||
],
|
||||
) -> dict[str, Any]:
|
||||
del principal
|
||||
return load_stage(9).run(
|
||||
context(body.corpus),
|
||||
question=body.question,
|
||||
top_k=body.top_k,
|
||||
)
|
||||
|
||||
@app.post("/internal/knowledge-graph/answer")
|
||||
def answer(
|
||||
body: QueryRequest,
|
||||
principal: Annotated[
|
||||
Principal,
|
||||
Depends(require_permissions(Permission.READ | Permission.EXECUTE)),
|
||||
],
|
||||
) -> dict[str, Any]:
|
||||
del principal
|
||||
return load_stage(10).run(
|
||||
context(body.corpus),
|
||||
question=body.question,
|
||||
top_k=body.top_k,
|
||||
)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
app = create_app()
|
||||
|
|
@ -0,0 +1,56 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from enum import IntFlag
|
||||
from typing import Any
|
||||
|
||||
import jwt
|
||||
|
||||
from graphrag_platform.config import Settings
|
||||
|
||||
|
||||
class Permission(IntFlag):
|
||||
READ = 1
|
||||
ADD = 2
|
||||
EDIT = 4
|
||||
DELETE = 8
|
||||
EXECUTE = 16
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Principal:
|
||||
subject: str
|
||||
permission_mask: int
|
||||
correlation_id: str
|
||||
claims: dict[str, Any]
|
||||
|
||||
def has(self, required: Permission) -> bool:
|
||||
return (self.permission_mask & int(required)) == int(required)
|
||||
|
||||
|
||||
def decode_internal_token(token: str, settings: Settings) -> Principal:
|
||||
claims = jwt.decode(
|
||||
token,
|
||||
settings.internal_jwt_secret,
|
||||
algorithms=["HS256"],
|
||||
issuer=settings.internal_jwt_issuer,
|
||||
audience=settings.internal_jwt_audience,
|
||||
options={"require": ["exp", "iat", "iss", "aud", "sub", "features"]},
|
||||
)
|
||||
features = claims.get("features")
|
||||
if not isinstance(features, dict):
|
||||
raise jwt.InvalidTokenError("features claim must be an object")
|
||||
raw_mask = features.get(settings.internal_feature_name)
|
||||
if not isinstance(raw_mask, int):
|
||||
raise jwt.InvalidTokenError(
|
||||
f"missing integer feature claim: {settings.internal_feature_name}"
|
||||
)
|
||||
correlation_id = str(claims.get("correlation_id") or claims.get("jti") or "")
|
||||
if not correlation_id:
|
||||
raise jwt.InvalidTokenError("correlation_id or jti is required")
|
||||
return Principal(
|
||||
subject=str(claims["sub"]),
|
||||
permission_mask=raw_mask,
|
||||
correlation_id=correlation_id,
|
||||
claims=claims,
|
||||
)
|
||||
|
|
@ -0,0 +1,111 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import sys
|
||||
from uuid import uuid4
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from graphrag_platform.config import Settings
|
||||
from graphrag_platform.observability import configure_logging
|
||||
from graphrag_platform.orchestrator import execute_run, submit_run
|
||||
from graphrag_platform.runtime import StageContext
|
||||
from graphrag_platform.stage_loader import STAGE_MODULES, load_stage
|
||||
|
||||
StageRunner = Callable[[StageContext, argparse.Namespace], dict[str, Any]]
|
||||
|
||||
|
||||
def create_parser(fixed_stage: int | None) -> argparse.ArgumentParser:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Run a KnowledgeGraph GraphRAG pipeline stage."
|
||||
)
|
||||
if fixed_stage is None:
|
||||
parser.add_argument("stage", type=int, choices=range(1, 11))
|
||||
parser.add_argument("--env-file", type=Path, default=Path(".env"))
|
||||
parser.add_argument("--database-url", help="Override DATABASE_URL.")
|
||||
parser.add_argument("--corpus", help="Override CORPUS_NAME.")
|
||||
parser.add_argument("--input-jsonl", type=Path)
|
||||
parser.add_argument("--model-file", type=Path)
|
||||
parser.add_argument("--question")
|
||||
parser.add_argument("--top-k", type=int)
|
||||
parser.add_argument("--reset", action="store_true")
|
||||
return parser
|
||||
|
||||
|
||||
def create_run_parser() -> argparse.ArgumentParser:
|
||||
parser = create_parser(1)
|
||||
parser.description = "Run a durable KnowledgeGraph build pipeline."
|
||||
parser.add_argument("--from-stage", type=int, choices=range(1, 9), default=1)
|
||||
parser.add_argument("--through-stage", type=int, choices=range(1, 9), default=8)
|
||||
parser.add_argument("--requested-by", default="local-cli")
|
||||
parser.add_argument("--correlation-id", default=None)
|
||||
return parser
|
||||
|
||||
|
||||
def create_context(
|
||||
args: argparse.Namespace, stage: int, *, apply_reset: bool = True
|
||||
) -> StageContext:
|
||||
overrides: dict[str, Any] = {}
|
||||
if args.database_url:
|
||||
overrides["database_url"] = args.database_url
|
||||
if args.corpus:
|
||||
overrides["corpus_name"] = args.corpus
|
||||
settings = Settings(_env_file=args.env_file, **overrides)
|
||||
context = StageContext.create(settings)
|
||||
context.initialize()
|
||||
if args.reset and apply_reset:
|
||||
context.storage.clear_from_stage(settings.corpus_name, stage)
|
||||
return context
|
||||
|
||||
|
||||
def run(stage: int, args: argparse.Namespace) -> dict[str, Any]:
|
||||
context = create_context(args, stage)
|
||||
module = load_stage(stage)
|
||||
return module.run_from_args(context, args)
|
||||
|
||||
|
||||
def stage_main(stage: int, runner: StageRunner) -> None:
|
||||
args = create_parser(stage).parse_args()
|
||||
context = create_context(args, stage)
|
||||
configure_logging(context.settings.log_level)
|
||||
result = runner(context, args)
|
||||
print(json.dumps(result, indent=2, ensure_ascii=False))
|
||||
|
||||
|
||||
def main_all() -> None:
|
||||
if len(sys.argv) > 1 and sys.argv[1] == "run":
|
||||
args = create_run_parser().parse_args(sys.argv[2:])
|
||||
context = create_context(args, args.from_stage, apply_reset=False)
|
||||
configure_logging(context.settings.log_level)
|
||||
run_id = submit_run(
|
||||
context,
|
||||
from_stage=args.from_stage,
|
||||
through_stage=args.through_stage,
|
||||
payload={
|
||||
"input_jsonl": str(args.input_jsonl) if args.input_jsonl else None,
|
||||
"model_file": str(args.model_file) if args.model_file else None,
|
||||
"reset": args.reset,
|
||||
},
|
||||
requested_by=args.requested_by,
|
||||
correlation_id=args.correlation_id or str(uuid4()),
|
||||
permission_mask=31,
|
||||
)
|
||||
outputs = execute_run(context, run_id)
|
||||
print(json.dumps({"run_id": run_id, "outputs": outputs}, indent=2))
|
||||
return
|
||||
args = create_parser(None).parse_args()
|
||||
configure_logging(Settings(_env_file=args.env_file).log_level)
|
||||
result = run(args.stage, args)
|
||||
print(
|
||||
json.dumps(
|
||||
{
|
||||
"stage": args.stage,
|
||||
"name": STAGE_MODULES[args.stage].removeprefix(f"{args.stage:02d}_"),
|
||||
**result,
|
||||
},
|
||||
indent=2,
|
||||
ensure_ascii=False,
|
||||
)
|
||||
)
|
||||
|
|
@ -0,0 +1,126 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Annotated, Any
|
||||
|
||||
from pydantic import Field, field_validator
|
||||
from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict
|
||||
|
||||
|
||||
class Settings(BaseSettings):
|
||||
"""Runtime configuration loaded from environment variables or a .env file."""
|
||||
|
||||
model_config = SettingsConfigDict(
|
||||
env_file=".env",
|
||||
env_file_encoding="utf-8",
|
||||
extra="ignore",
|
||||
case_sensitive=False,
|
||||
)
|
||||
|
||||
database_url: str = (
|
||||
"postgresql+psycopg://knowledgegraph_app:change-me"
|
||||
"@localhost:5432/knowledgegraph"
|
||||
)
|
||||
source_database_url: str | None = None
|
||||
source_table_name: str = "external_documents"
|
||||
analytics_database_url: str | None = None
|
||||
auto_create_schema: bool = False
|
||||
|
||||
corpus_name: str = "default-corpus"
|
||||
|
||||
entity_types: Annotated[list[str], NoDecode] = Field(
|
||||
default_factory=lambda: [
|
||||
"Person",
|
||||
"Organization",
|
||||
"Product",
|
||||
"Concept",
|
||||
"Event",
|
||||
"Location",
|
||||
]
|
||||
)
|
||||
relationship_types: Annotated[list[str], NoDecode] = Field(
|
||||
default_factory=lambda: [
|
||||
"AFFILIATED_WITH",
|
||||
"CREATED",
|
||||
"DEPENDS_ON",
|
||||
"MENTIONS",
|
||||
"PART_OF",
|
||||
"RELATED_TO",
|
||||
]
|
||||
)
|
||||
node_template: dict[str, Any] = Field(
|
||||
default_factory=lambda: {
|
||||
"canonical_name": "string",
|
||||
"entity_type": "one configured entity type",
|
||||
"description": "source-grounded string",
|
||||
"aliases": ["string"],
|
||||
}
|
||||
)
|
||||
edge_template: dict[str, Any] = Field(
|
||||
default_factory=lambda: {
|
||||
"source": "entity name",
|
||||
"target": "entity name",
|
||||
"relationship_type": "one configured relationship type",
|
||||
"description": "source-grounded string",
|
||||
}
|
||||
)
|
||||
extraction_instructions: str = (
|
||||
"Extract only entities and relationships explicitly supported by the supplied text."
|
||||
)
|
||||
|
||||
llm_base_url: str = "https://api.openai.com/v1"
|
||||
llm_api_key: str | None = None
|
||||
llm_chat_model: str = "gpt-4.1-mini"
|
||||
llm_embedding_model: str = "text-embedding-3-small"
|
||||
llm_embedding_dimensions: int = Field(default=1536, ge=1)
|
||||
llm_timeout_seconds: float = Field(default=60, gt=0)
|
||||
llm_max_retries: int = Field(default=4, ge=0, le=10)
|
||||
llm_retry_initial_seconds: float = Field(default=1, gt=0)
|
||||
llm_retry_max_seconds: float = Field(default=30, gt=0)
|
||||
|
||||
chunk_size: int = Field(default=1200, ge=200)
|
||||
chunk_overlap: int = Field(default=150, ge=0)
|
||||
community_resolution: float = Field(default=1.0, gt=0)
|
||||
retrieval_top_k: int = Field(default=8, ge=1)
|
||||
retrieval_graph_hops: int = Field(default=2, ge=0, le=5)
|
||||
retrieval_graph_limit: int = Field(default=50, ge=1)
|
||||
retrieval_hnsw_ef_search: int = Field(default=100, ge=1)
|
||||
log_level: str = "INFO"
|
||||
internal_jwt_secret: str = "replace-in-dev-test"
|
||||
internal_jwt_issuer: str = "knowledgegraph-bff"
|
||||
internal_jwt_audience: str = "knowledgegraph-service"
|
||||
internal_feature_name: str = "KnowledgeGraph"
|
||||
worker_poll_seconds: float = Field(default=2, gt=0)
|
||||
worker_stale_run_seconds: int = Field(default=3600, ge=60)
|
||||
artifact_dir: Path = Path("artifacts")
|
||||
|
||||
@field_validator(
|
||||
"entity_types",
|
||||
"relationship_types",
|
||||
mode="before",
|
||||
)
|
||||
@classmethod
|
||||
def parse_comma_separated_list(cls, value: object) -> object:
|
||||
if isinstance(value, str):
|
||||
return [item.strip() for item in value.split(",") if item.strip()]
|
||||
return value
|
||||
|
||||
@field_validator("node_template", "edge_template", mode="before")
|
||||
@classmethod
|
||||
def parse_json_object(cls, value: object) -> object:
|
||||
if isinstance(value, str):
|
||||
parsed = json.loads(value)
|
||||
if not isinstance(parsed, dict):
|
||||
raise ValueError("template must be a JSON object")
|
||||
return parsed
|
||||
return value
|
||||
|
||||
def model_definition(self) -> dict[str, Any]:
|
||||
return {
|
||||
"entity_types": self.entity_types,
|
||||
"relationship_types": self.relationship_types,
|
||||
"node_template": self.node_template,
|
||||
"edge_template": self.edge_template,
|
||||
"extraction_instructions": self.extraction_instructions,
|
||||
}
|
||||
|
|
@ -0,0 +1,153 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import random
|
||||
import time
|
||||
from collections.abc import Sequence
|
||||
from typing import Any, Callable, Protocol
|
||||
|
||||
import httpx
|
||||
|
||||
from graphrag_platform.config import Settings
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
_RETRYABLE_STATUS_CODES = {429, 500, 502, 503, 504}
|
||||
|
||||
|
||||
class LLMClient(Protocol):
|
||||
chat_model: str
|
||||
embedding_model: str
|
||||
|
||||
def complete_json(self, system: str, user: str) -> dict[str, Any]: ...
|
||||
|
||||
def complete_text(self, system: str, user: str) -> str: ...
|
||||
|
||||
def embed(self, texts: Sequence[str]) -> list[list[float]]: ...
|
||||
|
||||
|
||||
class OpenAICompatibleClient:
|
||||
def __init__(
|
||||
self,
|
||||
settings: Settings,
|
||||
transport: httpx.BaseTransport | None = None,
|
||||
sleeper: Callable[[float], None] = time.sleep,
|
||||
jitter: Callable[[], float] = random.random,
|
||||
) -> None:
|
||||
if not settings.llm_api_key:
|
||||
raise ValueError("LLM_API_KEY is required for LLM-backed stages")
|
||||
self.chat_model = settings.llm_chat_model
|
||||
self.embedding_model = settings.llm_embedding_model
|
||||
self._settings = settings
|
||||
self._sleeper = sleeper
|
||||
self._jitter = jitter
|
||||
self._client = httpx.Client(
|
||||
base_url=settings.llm_base_url.rstrip("/") + "/",
|
||||
timeout=settings.llm_timeout_seconds,
|
||||
headers={
|
||||
"Authorization": f"Bearer {settings.llm_api_key}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
transport=transport,
|
||||
)
|
||||
|
||||
def complete_json(self, system: str, user: str) -> dict[str, Any]:
|
||||
response = self._post(
|
||||
"chat/completions",
|
||||
json={
|
||||
"model": self.chat_model,
|
||||
"response_format": {"type": "json_object"},
|
||||
"temperature": 0,
|
||||
"messages": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
},
|
||||
)
|
||||
content = response.json()["choices"][0]["message"]["content"]
|
||||
parsed = json.loads(content)
|
||||
if not isinstance(parsed, dict):
|
||||
raise ValueError("LLM response must be a JSON object")
|
||||
return parsed
|
||||
|
||||
def complete_text(self, system: str, user: str) -> str:
|
||||
response = self._post(
|
||||
"chat/completions",
|
||||
json={
|
||||
"model": self.chat_model,
|
||||
"temperature": 0,
|
||||
"messages": [
|
||||
{"role": "system", "content": system},
|
||||
{"role": "user", "content": user},
|
||||
],
|
||||
},
|
||||
)
|
||||
return str(response.json()["choices"][0]["message"]["content"]).strip()
|
||||
|
||||
def embed(self, texts: Sequence[str]) -> list[list[float]]:
|
||||
if not texts:
|
||||
return []
|
||||
response = self._post(
|
||||
"embeddings",
|
||||
json={"model": self.embedding_model, "input": list(texts)},
|
||||
)
|
||||
rows = sorted(response.json()["data"], key=lambda row: row["index"])
|
||||
return [[float(value) for value in row["embedding"]] for row in rows]
|
||||
|
||||
def _post(self, path: str, *, json: dict[str, Any]) -> httpx.Response:
|
||||
attempts = self._settings.llm_max_retries + 1
|
||||
for attempt in range(1, attempts + 1):
|
||||
try:
|
||||
response = self._client.post(path, json=json)
|
||||
if response.status_code not in _RETRYABLE_STATUS_CODES:
|
||||
response.raise_for_status()
|
||||
return response
|
||||
if attempt == attempts:
|
||||
response.raise_for_status()
|
||||
delay = self._retry_delay(response, attempt)
|
||||
LOGGER.warning(
|
||||
"Retrying model provider request",
|
||||
extra={
|
||||
"event": "llm_retry",
|
||||
"attempt": attempt,
|
||||
"status_code": response.status_code,
|
||||
"delay_seconds": delay,
|
||||
"path": path,
|
||||
},
|
||||
)
|
||||
self._sleeper(delay)
|
||||
except (httpx.ConnectError, httpx.TimeoutException) as error:
|
||||
if attempt == attempts:
|
||||
raise
|
||||
delay = self._retry_delay(None, attempt)
|
||||
LOGGER.warning(
|
||||
"Retrying model provider request after transport error",
|
||||
extra={
|
||||
"event": "llm_retry",
|
||||
"attempt": attempt,
|
||||
"error_type": type(error).__name__,
|
||||
"delay_seconds": delay,
|
||||
"path": path,
|
||||
},
|
||||
)
|
||||
self._sleeper(delay)
|
||||
raise RuntimeError("model provider retry loop ended unexpectedly")
|
||||
|
||||
def _retry_delay(
|
||||
self, response: httpx.Response | None, attempt: int
|
||||
) -> float:
|
||||
if response is not None:
|
||||
retry_after = response.headers.get("Retry-After")
|
||||
if retry_after:
|
||||
try:
|
||||
return min(
|
||||
float(retry_after),
|
||||
self._settings.llm_retry_max_seconds,
|
||||
)
|
||||
except ValueError:
|
||||
pass
|
||||
exponential = self._settings.llm_retry_initial_seconds * (2 ** (attempt - 1))
|
||||
return min(
|
||||
exponential + self._jitter(),
|
||||
self._settings.llm_retry_max_seconds,
|
||||
)
|
||||
|
|
@ -0,0 +1,281 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
from uuid import uuid4
|
||||
|
||||
from sqlalchemy import (
|
||||
DateTime,
|
||||
Float,
|
||||
ForeignKey,
|
||||
Index,
|
||||
Integer,
|
||||
JSON,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
)
|
||||
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
||||
from pgvector.sqlalchemy import Vector
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
class Base(DeclarativeBase):
|
||||
pass
|
||||
|
||||
|
||||
class RawDocument(Base):
|
||||
__tablename__ = "raw_documents"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("corpus", "external_id", name="uq_raw_corpus_external_id"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
||||
external_id: Mapped[str] = mapped_column(String(500))
|
||||
source_schema_version: Mapped[str] = mapped_column(String(20))
|
||||
url: Mapped[str] = mapped_column(Text)
|
||||
content_type: Mapped[str | None] = mapped_column(String(200))
|
||||
status_code: Mapped[int | None] = mapped_column(Integer)
|
||||
body: Mapped[str] = mapped_column(Text)
|
||||
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
||||
collected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
|
||||
|
||||
|
||||
class Document(Base):
|
||||
__tablename__ = "documents"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("corpus", "url", name="uq_document_corpus_url"),
|
||||
UniqueConstraint("raw_document_id", name="uq_document_raw_document"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
raw_document_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("raw_documents.id", ondelete="SET NULL")
|
||||
)
|
||||
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
||||
url: Mapped[str] = mapped_column(Text)
|
||||
title: Mapped[str | None] = mapped_column(Text)
|
||||
author: Mapped[str | None] = mapped_column(Text)
|
||||
published_at: Mapped[str | None] = mapped_column(String(100))
|
||||
text: Mapped[str] = mapped_column(Text)
|
||||
content_hash: Mapped[str] = mapped_column(String(64), index=True)
|
||||
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
is_duplicate: Mapped[bool] = mapped_column(default=False)
|
||||
canonical_document_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("documents.id", ondelete="SET NULL")
|
||||
)
|
||||
|
||||
|
||||
class Chunk(Base):
|
||||
__tablename__ = "chunks"
|
||||
__table_args__ = (UniqueConstraint("document_id", "ordinal", name="uq_chunk_ordinal"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
document_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("documents.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
||||
ordinal: Mapped[int] = mapped_column(Integer)
|
||||
text: Mapped[str] = mapped_column(Text)
|
||||
content_hash: Mapped[str] = mapped_column(String(64), index=True)
|
||||
|
||||
|
||||
class KnowledgeModel(Base):
|
||||
__tablename__ = "knowledge_models"
|
||||
|
||||
corpus: Mapped[str] = mapped_column(String(200), primary_key=True)
|
||||
definition_json: Mapped[dict[str, Any]] = mapped_column(JSON)
|
||||
updated_at: Mapped[datetime] = mapped_column(
|
||||
DateTime(timezone=True), default=utc_now, onupdate=utc_now
|
||||
)
|
||||
|
||||
|
||||
class Extraction(Base):
|
||||
__tablename__ = "extractions"
|
||||
__table_args__ = (UniqueConstraint("chunk_id", name="uq_extraction_chunk"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
chunk_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("chunks.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
||||
payload_json: Mapped[dict[str, Any]] = mapped_column(JSON)
|
||||
model_name: Mapped[str] = mapped_column(String(200))
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
|
||||
|
||||
|
||||
class ResolvedEntity(Base):
|
||||
__tablename__ = "resolved_entities"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("corpus", "canonical_key", name="uq_resolved_entity_key"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
||||
canonical_key: Mapped[str] = mapped_column(String(500))
|
||||
canonical_name: Mapped[str] = mapped_column(Text)
|
||||
entity_type: Mapped[str] = mapped_column(String(100))
|
||||
description: Mapped[str] = mapped_column(Text, default="")
|
||||
aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list)
|
||||
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
|
||||
|
||||
|
||||
class ResolvedRelationship(Base):
|
||||
__tablename__ = "resolved_relationships"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"corpus",
|
||||
"source_key",
|
||||
"target_key",
|
||||
"relationship_type",
|
||||
name="uq_resolved_relationship",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
||||
source_key: Mapped[str] = mapped_column(String(500))
|
||||
target_key: Mapped[str] = mapped_column(String(500))
|
||||
relationship_type: Mapped[str] = mapped_column(String(100))
|
||||
description: Mapped[str] = mapped_column(Text, default="")
|
||||
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
|
||||
|
||||
|
||||
class Node(Base):
|
||||
__tablename__ = "nodes"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("corpus", "canonical_key", name="uq_node_canonical_key"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
||||
canonical_key: Mapped[str] = mapped_column(String(500))
|
||||
canonical_name: Mapped[str] = mapped_column(Text)
|
||||
entity_type: Mapped[str] = mapped_column(String(100), index=True)
|
||||
description: Mapped[str] = mapped_column(Text, default="")
|
||||
aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list)
|
||||
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
|
||||
|
||||
|
||||
class Edge(Base):
|
||||
__tablename__ = "edges"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"corpus",
|
||||
"source_node_id",
|
||||
"target_node_id",
|
||||
"relationship_type",
|
||||
name="uq_edge_identity",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
||||
source_node_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("nodes.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
target_node_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("nodes.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
relationship_type: Mapped[str] = mapped_column(String(100), index=True)
|
||||
description: Mapped[str] = mapped_column(Text, default="")
|
||||
weight: Mapped[float] = mapped_column(Float, default=1.0)
|
||||
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
|
||||
|
||||
|
||||
class Community(Base):
|
||||
__tablename__ = "communities"
|
||||
__table_args__ = (UniqueConstraint("corpus", "community_key", name="uq_community_key"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
||||
community_key: Mapped[str] = mapped_column(String(100))
|
||||
title: Mapped[str] = mapped_column(Text)
|
||||
summary: Mapped[str] = mapped_column(Text)
|
||||
node_ids_json: Mapped[list[int]] = mapped_column(JSON)
|
||||
|
||||
|
||||
class SearchItem(Base):
|
||||
__tablename__ = "retrieval_items"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("corpus", "item_type", "item_key", name="uq_search_item"),
|
||||
Index("ix_retrieval_items_corpus_type", "corpus", "item_type"),
|
||||
Index(
|
||||
"ix_retrieval_items_corpus_model",
|
||||
"corpus",
|
||||
"embedding_model",
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
||||
item_type: Mapped[str] = mapped_column(String(50))
|
||||
item_key: Mapped[str] = mapped_column(String(100))
|
||||
text: Mapped[str] = mapped_column(Text)
|
||||
source_url: Mapped[str | None] = mapped_column(Text)
|
||||
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
embedding: Mapped[list[float]] = mapped_column(
|
||||
Vector().with_variant(JSON(), "sqlite")
|
||||
)
|
||||
embedding_model: Mapped[str] = mapped_column(String(200))
|
||||
|
||||
|
||||
class PipelineRun(Base):
|
||||
__tablename__ = "pipeline_runs"
|
||||
__table_args__ = (
|
||||
Index("ix_pipeline_runs_status_created", "status", "created_at"),
|
||||
Index("ix_pipeline_runs_corpus_created", "corpus", "created_at"),
|
||||
)
|
||||
|
||||
id: Mapped[str] = mapped_column(
|
||||
String(36), primary_key=True, default=lambda: str(uuid4())
|
||||
)
|
||||
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
||||
status: Mapped[str] = mapped_column(String(30), index=True, default="queued")
|
||||
from_stage: Mapped[int] = mapped_column(Integer)
|
||||
through_stage: Mapped[int] = mapped_column(Integer)
|
||||
payload_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
configuration_hash: Mapped[str] = mapped_column(String(64))
|
||||
requested_by: Mapped[str] = mapped_column(String(500))
|
||||
correlation_id: Mapped[str] = mapped_column(String(200), index=True)
|
||||
feature_name: Mapped[str] = mapped_column(String(200), default="KnowledgeGraph")
|
||||
permission_mask: Mapped[int] = mapped_column(Integer)
|
||||
attempt: Mapped[int] = mapped_column(Integer, default=0)
|
||||
output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
error_message: Mapped[str | None] = mapped_column(Text)
|
||||
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
|
||||
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
||||
|
||||
class StageRun(Base):
|
||||
__tablename__ = "stage_runs"
|
||||
__table_args__ = (
|
||||
UniqueConstraint(
|
||||
"pipeline_run_id",
|
||||
"stage_number",
|
||||
"attempt",
|
||||
name="uq_stage_run_attempt",
|
||||
),
|
||||
Index("ix_stage_runs_pipeline_stage", "pipeline_run_id", "stage_number"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
pipeline_run_id: Mapped[str] = mapped_column(
|
||||
ForeignKey("pipeline_runs.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
stage_number: Mapped[int] = mapped_column(Integer)
|
||||
stage_name: Mapped[str] = mapped_column(String(200))
|
||||
status: Mapped[str] = mapped_column(String(30), default="running")
|
||||
attempt: Mapped[int] = mapped_column(Integer)
|
||||
input_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
||||
error_message: Mapped[str | None] = mapped_column(Text)
|
||||
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
|
||||
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
|
||||
class JsonFormatter(logging.Formatter):
|
||||
_reserved = set(logging.makeLogRecord({}).__dict__)
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
payload: dict[str, Any] = {
|
||||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"message": record.getMessage(),
|
||||
}
|
||||
for key, value in record.__dict__.items():
|
||||
if key not in self._reserved and key not in {"message", "asctime"}:
|
||||
payload[key] = value
|
||||
if record.exc_info:
|
||||
payload["exception"] = self.formatException(record.exc_info)
|
||||
return json.dumps(payload, ensure_ascii=False, default=str)
|
||||
|
||||
|
||||
def configure_logging(level: str = "INFO") -> None:
|
||||
handler = logging.StreamHandler()
|
||||
handler.setFormatter(JsonFormatter())
|
||||
root = logging.getLogger()
|
||||
root.handlers.clear()
|
||||
root.addHandler(handler)
|
||||
root.setLevel(level.upper())
|
||||
|
|
@ -0,0 +1,330 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import logging
|
||||
from datetime import datetime, timezone
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import select, text, update
|
||||
|
||||
from graphrag_platform.models import PipelineRun, StageRun
|
||||
from graphrag_platform.runtime import StageContext
|
||||
from graphrag_platform.stage_loader import STAGE_MODULES, load_stage
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
BUILD_STAGES = range(1, 9)
|
||||
|
||||
|
||||
class RunDeferredError(RuntimeError):
|
||||
pass
|
||||
|
||||
|
||||
def utc_now() -> datetime:
|
||||
return datetime.now(timezone.utc)
|
||||
|
||||
|
||||
def configuration_hash(context: StageContext) -> str:
|
||||
payload = context.settings.model_dump(
|
||||
mode="json",
|
||||
exclude={"llm_api_key", "internal_jwt_secret"},
|
||||
)
|
||||
encoded = json.dumps(payload, sort_keys=True, separators=(",", ":"))
|
||||
return hashlib.sha256(encoded.encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def submit_run(
|
||||
context: StageContext,
|
||||
*,
|
||||
from_stage: int,
|
||||
through_stage: int,
|
||||
payload: dict[str, Any],
|
||||
requested_by: str,
|
||||
correlation_id: str,
|
||||
permission_mask: int,
|
||||
) -> str:
|
||||
if from_stage not in BUILD_STAGES or through_stage not in BUILD_STAGES:
|
||||
raise ValueError("pipeline jobs support build stages 1 through 8")
|
||||
if from_stage > through_stage:
|
||||
raise ValueError("from_stage must not exceed through_stage")
|
||||
run = PipelineRun(
|
||||
corpus=context.settings.corpus_name,
|
||||
status="queued",
|
||||
from_stage=from_stage,
|
||||
through_stage=through_stage,
|
||||
payload_json=payload,
|
||||
configuration_hash=configuration_hash(context),
|
||||
requested_by=requested_by,
|
||||
correlation_id=correlation_id,
|
||||
feature_name=context.settings.internal_feature_name,
|
||||
permission_mask=permission_mask,
|
||||
)
|
||||
with context.storage.session() as session:
|
||||
session.add(run)
|
||||
session.flush()
|
||||
run_id = run.id
|
||||
LOGGER.info(
|
||||
"Pipeline run queued",
|
||||
extra={
|
||||
"event": "pipeline_queued",
|
||||
"run_id": run_id,
|
||||
"corpus": context.settings.corpus_name,
|
||||
"from_stage": from_stage,
|
||||
"through_stage": through_stage,
|
||||
"correlation_id": correlation_id,
|
||||
},
|
||||
)
|
||||
return run_id
|
||||
|
||||
|
||||
def claim_next_run(context: StageContext) -> str | None:
|
||||
with context.storage.session() as session:
|
||||
statement = (
|
||||
select(PipelineRun)
|
||||
.where(PipelineRun.status == "queued")
|
||||
.order_by(PipelineRun.created_at)
|
||||
.limit(1)
|
||||
)
|
||||
if context.storage.engine.dialect.name == "postgresql":
|
||||
statement = statement.with_for_update(skip_locked=True)
|
||||
run = session.scalar(statement)
|
||||
if run is None:
|
||||
return None
|
||||
run.status = "running"
|
||||
run.attempt += 1
|
||||
run.started_at = run.started_at or utc_now()
|
||||
run.heartbeat_at = utc_now()
|
||||
session.flush()
|
||||
return run.id
|
||||
|
||||
|
||||
def execute_run(context: StageContext, run_id: str) -> dict[str, Any]:
|
||||
with context.storage.session() as session:
|
||||
run = session.get(PipelineRun, run_id)
|
||||
if run is None:
|
||||
raise ValueError(f"pipeline run does not exist: {run_id}")
|
||||
if run.status == "queued":
|
||||
run.status = "running"
|
||||
run.attempt += 1
|
||||
run.started_at = run.started_at or utc_now()
|
||||
if run.status != "running":
|
||||
raise ValueError(f"pipeline run {run_id} is {run.status}, not running")
|
||||
from_stage = run.from_stage
|
||||
through_stage = run.through_stage
|
||||
payload = dict(run.payload_json)
|
||||
attempt = run.attempt
|
||||
execution_context = context.for_corpus(run.corpus)
|
||||
corpus = run.corpus
|
||||
|
||||
lock_connection = None
|
||||
if context.storage.engine.dialect.name == "postgresql":
|
||||
lock_connection = context.storage.engine.connect()
|
||||
acquired = lock_connection.scalar(
|
||||
text("SELECT pg_try_advisory_lock(hashtext(:key))"),
|
||||
{"key": f"knowledgegraph:{corpus}"},
|
||||
)
|
||||
if not acquired:
|
||||
lock_connection.close()
|
||||
with context.storage.session() as session:
|
||||
run = session.get(PipelineRun, run_id)
|
||||
if run is not None:
|
||||
run.status = "queued"
|
||||
run.error_message = "Deferred because the corpus is already running"
|
||||
raise RunDeferredError(f"corpus already has an active run: {corpus}")
|
||||
|
||||
outputs: dict[str, Any] = {}
|
||||
try:
|
||||
if payload.get("reset") and attempt == 1:
|
||||
execution_context.storage.clear_from_stage(corpus, from_stage)
|
||||
for stage_number in range(from_stage, through_stage + 1):
|
||||
previous_output = _successful_stage_output(
|
||||
context, run_id, stage_number
|
||||
)
|
||||
if previous_output is not None:
|
||||
outputs[str(stage_number)] = previous_output
|
||||
continue
|
||||
stage_name = STAGE_MODULES[stage_number]
|
||||
stage_run_id = _start_stage_run(
|
||||
context,
|
||||
run_id=run_id,
|
||||
stage_number=stage_number,
|
||||
stage_name=stage_name,
|
||||
attempt=attempt,
|
||||
payload=payload,
|
||||
)
|
||||
try:
|
||||
result = _execute_stage(execution_context, stage_number, payload)
|
||||
except Exception as error:
|
||||
_finish_stage_run(
|
||||
context,
|
||||
stage_run_id,
|
||||
status="failed",
|
||||
error_message=str(error),
|
||||
)
|
||||
raise
|
||||
_finish_stage_run(
|
||||
context,
|
||||
stage_run_id,
|
||||
status="succeeded",
|
||||
output=result,
|
||||
)
|
||||
outputs[str(stage_number)] = result
|
||||
_checkpoint(context, run_id, outputs)
|
||||
with context.storage.session() as session:
|
||||
run = session.get(PipelineRun, run_id)
|
||||
if run is None:
|
||||
raise RuntimeError(f"pipeline run disappeared: {run_id}")
|
||||
run.status = "succeeded"
|
||||
run.output_json = outputs
|
||||
run.finished_at = utc_now()
|
||||
run.heartbeat_at = utc_now()
|
||||
LOGGER.info(
|
||||
"Pipeline run succeeded",
|
||||
extra={"event": "pipeline_succeeded", "run_id": run_id},
|
||||
)
|
||||
return outputs
|
||||
except Exception as error:
|
||||
with context.storage.session() as session:
|
||||
run = session.get(PipelineRun, run_id)
|
||||
if run is not None:
|
||||
run.status = "failed"
|
||||
run.error_message = str(error)
|
||||
run.finished_at = utc_now()
|
||||
run.heartbeat_at = utc_now()
|
||||
LOGGER.exception(
|
||||
"Pipeline run failed",
|
||||
extra={"event": "pipeline_failed", "run_id": run_id},
|
||||
)
|
||||
raise
|
||||
finally:
|
||||
if lock_connection is not None:
|
||||
lock_connection.execute(
|
||||
text("SELECT pg_advisory_unlock(hashtext(:key))"),
|
||||
{"key": f"knowledgegraph:{corpus}"},
|
||||
)
|
||||
lock_connection.close()
|
||||
|
||||
|
||||
def requeue_failed_run(context: StageContext, run_id: str) -> None:
|
||||
with context.storage.session() as session:
|
||||
run = session.get(PipelineRun, run_id)
|
||||
if run is None:
|
||||
raise ValueError(f"pipeline run does not exist: {run_id}")
|
||||
if run.status != "failed":
|
||||
raise ValueError("only failed pipeline runs can be requeued")
|
||||
run.status = "queued"
|
||||
run.error_message = None
|
||||
run.finished_at = None
|
||||
|
||||
|
||||
def recover_stale_runs(context: StageContext) -> int:
|
||||
threshold = utc_now().timestamp() - context.settings.worker_stale_run_seconds
|
||||
cutoff = datetime.fromtimestamp(threshold, tz=timezone.utc)
|
||||
with context.storage.session() as session:
|
||||
result = session.execute(
|
||||
update(PipelineRun)
|
||||
.where(
|
||||
PipelineRun.status == "running",
|
||||
PipelineRun.heartbeat_at < cutoff,
|
||||
)
|
||||
.values(
|
||||
status="queued",
|
||||
error_message="Recovered after a stale worker lease",
|
||||
finished_at=None,
|
||||
)
|
||||
)
|
||||
recovered = int(result.rowcount or 0)
|
||||
if recovered:
|
||||
LOGGER.warning(
|
||||
"Recovered stale pipeline runs",
|
||||
extra={"event": "pipeline_stale_recovered", "count": recovered},
|
||||
)
|
||||
return recovered
|
||||
|
||||
|
||||
def _execute_stage(
|
||||
context: StageContext, stage_number: int, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
module = load_stage(stage_number)
|
||||
if stage_number == 1:
|
||||
input_path = payload.get("input_jsonl")
|
||||
return module.run(
|
||||
context,
|
||||
input_jsonl=Path(input_path) if input_path else None,
|
||||
)
|
||||
if stage_number == 3:
|
||||
model_path = payload.get("model_file")
|
||||
return module.run(
|
||||
context,
|
||||
model_file=Path(model_path) if model_path else None,
|
||||
)
|
||||
return module.run(context)
|
||||
|
||||
|
||||
def _start_stage_run(
|
||||
context: StageContext,
|
||||
*,
|
||||
run_id: str,
|
||||
stage_number: int,
|
||||
stage_name: str,
|
||||
attempt: int,
|
||||
payload: dict[str, Any],
|
||||
) -> int:
|
||||
with context.storage.session() as session:
|
||||
stage_run = StageRun(
|
||||
pipeline_run_id=run_id,
|
||||
stage_number=stage_number,
|
||||
stage_name=stage_name,
|
||||
status="running",
|
||||
attempt=attempt,
|
||||
input_json=payload,
|
||||
)
|
||||
session.add(stage_run)
|
||||
session.flush()
|
||||
return stage_run.id
|
||||
|
||||
|
||||
def _finish_stage_run(
|
||||
context: StageContext,
|
||||
stage_run_id: int,
|
||||
*,
|
||||
status: str,
|
||||
output: dict[str, Any] | None = None,
|
||||
error_message: str | None = None,
|
||||
) -> None:
|
||||
with context.storage.session() as session:
|
||||
stage_run = session.get(StageRun, stage_run_id)
|
||||
if stage_run is None:
|
||||
raise RuntimeError(f"stage run disappeared: {stage_run_id}")
|
||||
stage_run.status = status
|
||||
stage_run.output_json = output or {}
|
||||
stage_run.error_message = error_message
|
||||
stage_run.finished_at = utc_now()
|
||||
|
||||
|
||||
def _checkpoint(
|
||||
context: StageContext, run_id: str, outputs: dict[str, Any]
|
||||
) -> None:
|
||||
with context.storage.session() as session:
|
||||
run = session.get(PipelineRun, run_id)
|
||||
if run is not None:
|
||||
run.output_json = outputs
|
||||
run.heartbeat_at = utc_now()
|
||||
|
||||
|
||||
def _successful_stage_output(
|
||||
context: StageContext, run_id: str, stage_number: int
|
||||
) -> dict[str, Any] | None:
|
||||
with context.storage.session() as session:
|
||||
stage_run = session.scalar(
|
||||
select(StageRun)
|
||||
.where(
|
||||
StageRun.pipeline_run_id == run_id,
|
||||
StageRun.stage_number == stage_number,
|
||||
StageRun.status == "succeeded",
|
||||
)
|
||||
.order_by(StageRun.attempt.desc())
|
||||
.limit(1)
|
||||
)
|
||||
return dict(stage_run.output_json) if stage_run is not None else None
|
||||
|
|
@ -0,0 +1,72 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from graphrag_platform.config import Settings
|
||||
from graphrag_platform.llm import LLMClient
|
||||
from graphrag_platform.runtime import StageContext
|
||||
from graphrag_platform.stage_loader import load_stage
|
||||
from graphrag_platform.storage import Storage
|
||||
|
||||
|
||||
class KnowledgeGraphPipeline:
|
||||
"""Compatibility facade over independently executable numbered stages."""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings: Settings,
|
||||
*,
|
||||
storage: Storage | None = None,
|
||||
llm: LLMClient | None = None,
|
||||
) -> None:
|
||||
self.context = StageContext.create(settings, storage=storage, llm=llm)
|
||||
|
||||
@property
|
||||
def settings(self) -> Settings:
|
||||
return self.context.settings
|
||||
|
||||
@property
|
||||
def storage(self) -> Storage:
|
||||
return self.context.storage
|
||||
|
||||
def initialize(self) -> None:
|
||||
self.context.initialize()
|
||||
|
||||
def stage_01_normalize(
|
||||
self, *, input_jsonl: Path | None = None
|
||||
) -> dict[str, int]:
|
||||
return load_stage(1).run(self.context, input_jsonl=input_jsonl)
|
||||
|
||||
def stage_02_clean_and_chunk(self) -> dict[str, int]:
|
||||
return load_stage(2).run(self.context)
|
||||
|
||||
def stage_03_define_model(
|
||||
self, model_file: Path | None = None
|
||||
) -> dict[str, int]:
|
||||
return load_stage(3).run(self.context, model_file=model_file)
|
||||
|
||||
def stage_04_extract(self) -> dict[str, int]:
|
||||
return load_stage(4).run(self.context)
|
||||
|
||||
def stage_05_reconcile(self) -> dict[str, int]:
|
||||
return load_stage(5).run(self.context)
|
||||
|
||||
def stage_06_build_graph(self) -> dict[str, int]:
|
||||
return load_stage(6).run(self.context)
|
||||
|
||||
def stage_07_build_communities(self) -> dict[str, int]:
|
||||
return load_stage(7).run(self.context)
|
||||
|
||||
def stage_08_build_retrieval_index(self) -> dict[str, int]:
|
||||
return load_stage(8).run(self.context)
|
||||
|
||||
def stage_09_retrieve(
|
||||
self, question: str, *, top_k: int | None = None
|
||||
) -> dict[str, Any]:
|
||||
return load_stage(9).run(self.context, question=question, top_k=top_k)
|
||||
|
||||
def stage_10_answer(
|
||||
self, question: str, *, top_k: int | None = None
|
||||
) -> dict[str, Any]:
|
||||
return load_stage(10).run(self.context, question=question, top_k=top_k)
|
||||
|
|
@ -0,0 +1,50 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
|
||||
from graphrag_platform.config import Settings
|
||||
from graphrag_platform.llm import LLMClient, OpenAICompatibleClient
|
||||
from graphrag_platform.storage import Storage
|
||||
|
||||
|
||||
@dataclass
|
||||
class StageContext:
|
||||
settings: Settings
|
||||
storage: Storage
|
||||
_llm: LLMClient | None = None
|
||||
|
||||
@classmethod
|
||||
def create(
|
||||
cls,
|
||||
settings: Settings,
|
||||
*,
|
||||
storage: Storage | None = None,
|
||||
llm: LLMClient | None = None,
|
||||
) -> StageContext:
|
||||
return cls(
|
||||
settings=settings,
|
||||
storage=storage or Storage(settings.database_url),
|
||||
_llm=llm,
|
||||
)
|
||||
|
||||
@property
|
||||
def llm(self) -> LLMClient:
|
||||
if self._llm is None:
|
||||
self._llm = OpenAICompatibleClient(self.settings)
|
||||
return self._llm
|
||||
|
||||
def initialize(self) -> None:
|
||||
if self.settings.auto_create_schema:
|
||||
self.storage.create_schema()
|
||||
else:
|
||||
self.storage.verify_schema()
|
||||
(self.settings.artifact_dir / self.settings.corpus_name).mkdir(
|
||||
parents=True, exist_ok=True
|
||||
)
|
||||
|
||||
def for_corpus(self, corpus: str) -> StageContext:
|
||||
return StageContext.create(
|
||||
self.settings.model_copy(update={"corpus_name": corpus}),
|
||||
storage=self.storage,
|
||||
llm=self._llm,
|
||||
)
|
||||
|
|
@ -0,0 +1,106 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import re
|
||||
from collections.abc import Iterator
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
from typing import Any, Protocol
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field, HttpUrl, field_validator
|
||||
from sqlalchemy import MetaData, Table, create_engine, select
|
||||
|
||||
_TABLE_NAME_RE = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*$")
|
||||
|
||||
|
||||
class SourceRecord(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
schema_version: str = "1.0"
|
||||
external_id: str = Field(min_length=1, max_length=500)
|
||||
corpus: str | None = Field(default=None, min_length=1, max_length=200)
|
||||
url: HttpUrl
|
||||
body: str = Field(min_length=1)
|
||||
content_type: str = "text/html"
|
||||
status_code: int = Field(default=200, ge=100, le=599)
|
||||
retrieved_at: datetime
|
||||
metadata: dict[str, Any] = Field(default_factory=dict)
|
||||
|
||||
@field_validator("schema_version")
|
||||
@classmethod
|
||||
def require_supported_schema(cls, value: str) -> str:
|
||||
if value != "1.0":
|
||||
raise ValueError(f"unsupported source schema version: {value}")
|
||||
return value
|
||||
|
||||
|
||||
class SourceReader(Protocol):
|
||||
def records(self) -> Iterator[SourceRecord]: ...
|
||||
|
||||
|
||||
class JsonLinesSourceReader:
|
||||
def __init__(self, path: Path) -> None:
|
||||
self.path = path
|
||||
|
||||
def records(self) -> Iterator[SourceRecord]:
|
||||
with self.path.open("r", encoding="utf-8") as stream:
|
||||
for line_number, line in enumerate(stream, start=1):
|
||||
if not line.strip():
|
||||
continue
|
||||
try:
|
||||
payload = json.loads(line)
|
||||
yield SourceRecord.model_validate(payload)
|
||||
except (json.JSONDecodeError, ValueError) as error:
|
||||
raise ValueError(
|
||||
f"{self.path}:{line_number}: invalid source record: {error}"
|
||||
) from error
|
||||
|
||||
|
||||
class PostgreSqlSourceReader:
|
||||
def __init__(self, database_url: str, table_name: str) -> None:
|
||||
if not database_url.startswith("postgresql"):
|
||||
raise ValueError("SOURCE_DATABASE_URL must use PostgreSQL")
|
||||
if not _TABLE_NAME_RE.fullmatch(table_name):
|
||||
raise ValueError("SOURCE_TABLE_NAME must be an unqualified SQL identifier")
|
||||
self.engine = create_engine(database_url, pool_pre_ping=True)
|
||||
self.table_name = table_name
|
||||
|
||||
def records(self) -> Iterator[SourceRecord]:
|
||||
metadata = MetaData()
|
||||
table = Table(self.table_name, metadata, autoload_with=self.engine)
|
||||
required = {
|
||||
"schema_version",
|
||||
"external_id",
|
||||
"url",
|
||||
"body",
|
||||
"retrieved_at",
|
||||
}
|
||||
missing = required - set(table.columns.keys())
|
||||
if missing:
|
||||
raise ValueError(
|
||||
f"source table {self.table_name} is missing columns: "
|
||||
f"{', '.join(sorted(missing))}"
|
||||
)
|
||||
with self.engine.connect() as connection:
|
||||
for row in connection.execute(select(table).order_by(table.c.external_id)):
|
||||
payload = {
|
||||
key: value
|
||||
for key, value in row._mapping.items()
|
||||
if key in SourceRecord.model_fields
|
||||
}
|
||||
yield SourceRecord.model_validate(payload)
|
||||
|
||||
|
||||
def source_reader(
|
||||
*,
|
||||
input_jsonl: Path | None,
|
||||
source_database_url: str | None,
|
||||
source_table_name: str,
|
||||
) -> SourceReader:
|
||||
if input_jsonl is not None and source_database_url:
|
||||
raise ValueError("provide either --input-jsonl or SOURCE_DATABASE_URL, not both")
|
||||
if input_jsonl is not None:
|
||||
return JsonLinesSourceReader(input_jsonl)
|
||||
if source_database_url:
|
||||
return PostgreSqlSourceReader(source_database_url, source_table_name)
|
||||
raise ValueError("provide --input-jsonl or configure SOURCE_DATABASE_URL")
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from importlib import import_module
|
||||
from types import ModuleType
|
||||
|
||||
STAGE_MODULES = {
|
||||
1: "01_normalize_content",
|
||||
2: "02_clean_content",
|
||||
3: "03_define_knowledge_model",
|
||||
4: "04_extract_knowledge",
|
||||
5: "05_reconcile_knowledge",
|
||||
6: "06_build_knowledge_graph",
|
||||
7: "07_build_communities",
|
||||
8: "08_build_retrieval_index",
|
||||
9: "09_query_graphrag",
|
||||
10: "10_generate_answer",
|
||||
}
|
||||
|
||||
|
||||
def load_stage(stage: int) -> ModuleType:
|
||||
try:
|
||||
module_name = STAGE_MODULES[stage]
|
||||
except KeyError as error:
|
||||
raise ValueError(f"unsupported stage {stage}") from error
|
||||
return import_module(module_name)
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterator
|
||||
from contextlib import contextmanager
|
||||
from typing import Any, TypeVar
|
||||
|
||||
from sqlalchemy import create_engine, delete, event, inspect, select, text
|
||||
from sqlalchemy.engine import Engine
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from graphrag_platform.models import (
|
||||
Base,
|
||||
Chunk,
|
||||
Community,
|
||||
Document,
|
||||
Edge,
|
||||
Extraction,
|
||||
KnowledgeModel,
|
||||
Node,
|
||||
RawDocument,
|
||||
ResolvedEntity,
|
||||
ResolvedRelationship,
|
||||
SearchItem,
|
||||
)
|
||||
|
||||
ModelT = TypeVar("ModelT", bound=Base)
|
||||
|
||||
|
||||
class Storage:
|
||||
def __init__(self, database_url: str) -> None:
|
||||
connect_args = {"check_same_thread": False} if database_url.startswith("sqlite") else {}
|
||||
self.engine: Engine = create_engine(
|
||||
database_url,
|
||||
future=True,
|
||||
pool_pre_ping=True,
|
||||
connect_args=connect_args,
|
||||
)
|
||||
if database_url.startswith("sqlite"):
|
||||
event.listen(self.engine, "connect", self._enable_sqlite_foreign_keys)
|
||||
|
||||
@staticmethod
|
||||
def _enable_sqlite_foreign_keys(
|
||||
dbapi_connection: Any, connection_record: Any
|
||||
) -> None:
|
||||
del connection_record
|
||||
cursor = dbapi_connection.cursor()
|
||||
cursor.execute("PRAGMA foreign_keys=ON")
|
||||
cursor.close()
|
||||
|
||||
def create_schema(self) -> None:
|
||||
if self.engine.dialect.name == "postgresql":
|
||||
with self.engine.begin() as connection:
|
||||
connection.execute(text("CREATE EXTENSION IF NOT EXISTS vector"))
|
||||
Base.metadata.create_all(self.engine)
|
||||
|
||||
def verify_schema(self) -> None:
|
||||
tables = set(inspect(self.engine).get_table_names())
|
||||
required = {"pipeline_runs", "stage_runs", "documents", "retrieval_items"}
|
||||
missing = required - tables
|
||||
if missing:
|
||||
raise RuntimeError(
|
||||
"database migrations are required; missing tables: "
|
||||
f"{', '.join(sorted(missing))}"
|
||||
)
|
||||
if self.engine.dialect.name == "postgresql":
|
||||
with self.engine.connect() as connection:
|
||||
extension = connection.scalar(
|
||||
text(
|
||||
"SELECT extname FROM pg_extension "
|
||||
"WHERE extname = 'vector'"
|
||||
)
|
||||
)
|
||||
if extension != "vector":
|
||||
raise RuntimeError("database migrations are required: vector extension")
|
||||
|
||||
@property
|
||||
def supports_vector_search(self) -> bool:
|
||||
return self.engine.dialect.name == "postgresql"
|
||||
|
||||
def ensure_vector_index(self, dimensions: int) -> None:
|
||||
if not self.supports_vector_search:
|
||||
return
|
||||
index_name = f"ix_retrieval_items_embedding_hnsw_{dimensions}"
|
||||
statement = f"""
|
||||
CREATE INDEX IF NOT EXISTS {index_name}
|
||||
ON retrieval_items
|
||||
USING hnsw ((embedding::vector({dimensions})) vector_cosine_ops)
|
||||
WHERE vector_dims(embedding) = {dimensions}
|
||||
"""
|
||||
with self.engine.begin() as connection:
|
||||
connection.execute(text(statement))
|
||||
|
||||
@contextmanager
|
||||
def session(self) -> Iterator[Session]:
|
||||
with Session(self.engine) as session:
|
||||
try:
|
||||
yield session
|
||||
session.commit()
|
||||
except Exception:
|
||||
session.rollback()
|
||||
raise
|
||||
|
||||
def upsert(
|
||||
self,
|
||||
session: Session,
|
||||
model: type[ModelT],
|
||||
identity: dict[str, Any],
|
||||
values: dict[str, Any],
|
||||
) -> ModelT:
|
||||
instance = session.scalar(select(model).filter_by(**identity))
|
||||
if instance is None:
|
||||
instance = model(**identity, **values)
|
||||
session.add(instance)
|
||||
else:
|
||||
for key, value in values.items():
|
||||
setattr(instance, key, value)
|
||||
session.flush()
|
||||
return instance
|
||||
|
||||
def clear_from_stage(self, corpus: str, stage: int) -> None:
|
||||
stage_tables: list[tuple[int, type[Base]]] = [
|
||||
(8, SearchItem),
|
||||
(7, Community),
|
||||
(6, Edge),
|
||||
(6, Node),
|
||||
(5, ResolvedRelationship),
|
||||
(5, ResolvedEntity),
|
||||
(4, Extraction),
|
||||
(2, Chunk),
|
||||
(1, RawDocument),
|
||||
(1, Document),
|
||||
]
|
||||
with self.session() as session:
|
||||
for table_stage, model in stage_tables:
|
||||
if table_stage >= stage:
|
||||
session.execute(delete(model).where(model.corpus == corpus))
|
||||
if stage <= 3:
|
||||
session.execute(
|
||||
delete(KnowledgeModel).where(KnowledgeModel.corpus == corpus)
|
||||
)
|
||||
|
|
@ -0,0 +1,61 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import logging
|
||||
import time
|
||||
|
||||
from graphrag_platform.config import Settings
|
||||
from graphrag_platform.observability import configure_logging
|
||||
from graphrag_platform.orchestrator import (
|
||||
RunDeferredError,
|
||||
claim_next_run,
|
||||
execute_run,
|
||||
recover_stale_runs,
|
||||
)
|
||||
from graphrag_platform.runtime import StageContext
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def work_once(context: StageContext) -> bool:
|
||||
run_id = claim_next_run(context)
|
||||
if run_id is None:
|
||||
return False
|
||||
try:
|
||||
execute_run(context, run_id)
|
||||
except RunDeferredError:
|
||||
LOGGER.info(
|
||||
"Worker deferred a run for an active corpus",
|
||||
extra={"event": "worker_job_deferred", "run_id": run_id},
|
||||
)
|
||||
except Exception:
|
||||
LOGGER.exception(
|
||||
"Worker execution failed",
|
||||
extra={"event": "worker_job_failed", "run_id": run_id},
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
def run_worker(context: StageContext, *, once: bool = False) -> None:
|
||||
context.initialize()
|
||||
recover_stale_runs(context)
|
||||
while True:
|
||||
worked = work_once(context)
|
||||
if once:
|
||||
return
|
||||
if not worked:
|
||||
time.sleep(context.settings.worker_poll_seconds)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Run the KnowledgeGraph job worker.")
|
||||
parser.add_argument("--env-file", default=".env")
|
||||
parser.add_argument("--once", action="store_true")
|
||||
args = parser.parse_args()
|
||||
settings = Settings(_env_file=args.env_file)
|
||||
configure_logging(settings.log_level)
|
||||
run_worker(StageContext.create(settings), once=args.once)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
|
@ -0,0 +1,33 @@
|
|||
{
|
||||
"entity_types": [
|
||||
"Person",
|
||||
"Organization",
|
||||
"Product",
|
||||
"Concept",
|
||||
"Event",
|
||||
"Location"
|
||||
],
|
||||
"relationship_types": [
|
||||
"AFFILIATED_WITH",
|
||||
"CREATED",
|
||||
"DEPENDS_ON",
|
||||
"MENTIONS",
|
||||
"PART_OF",
|
||||
"RELATED_TO"
|
||||
],
|
||||
"node_template": {
|
||||
"canonical_name": "string",
|
||||
"entity_type": "one configured entity type",
|
||||
"description": "source-grounded string",
|
||||
"aliases": [
|
||||
"string"
|
||||
]
|
||||
},
|
||||
"edge_template": {
|
||||
"source": "entity name",
|
||||
"target": "entity name",
|
||||
"relationship_type": "one configured relationship type",
|
||||
"description": "source-grounded string"
|
||||
},
|
||||
"extraction_instructions": "Extract only entities and relationships explicitly supported by the supplied text."
|
||||
}
|
||||
|
|
@ -0,0 +1 @@
|
|||
{"schema_version":"1.0","external_id":"docs-example-article","corpus":"product-knowledge","url":"https://docs.example.com/article","content_type":"text/html","status_code":200,"retrieved_at":"2026-01-01T00:00:00Z","body":"<html><head><title>Article title</title></head><body><main><h1>Article title</h1><p>Article text collected with Web-Access.</p></main></body></html>","metadata":{"collector":"Web-Access","stored_by":"Data-Store"}}
|
||||
|
|
@ -0,0 +1,58 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from collections.abc import Sequence
|
||||
from typing import Any
|
||||
|
||||
|
||||
class FakeLLM:
|
||||
chat_model = "fake-chat-model"
|
||||
embedding_model = "fake-embedding-model"
|
||||
|
||||
def complete_json(self, system: str, user: str) -> dict[str, Any]:
|
||||
if "community" in system.casefold():
|
||||
return {
|
||||
"title": "Atlas product community",
|
||||
"summary": "Acme created Atlas, a graph retrieval product.",
|
||||
}
|
||||
return {
|
||||
"entities": [
|
||||
{
|
||||
"canonical_name": "Acme",
|
||||
"entity_type": "Organization",
|
||||
"description": "Acme is a technology organization.",
|
||||
"aliases": ["Acme Corp"],
|
||||
},
|
||||
{
|
||||
"canonical_name": "Atlas",
|
||||
"entity_type": "Product",
|
||||
"description": "Atlas is a graph retrieval product.",
|
||||
"aliases": [],
|
||||
},
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"source": "Acme",
|
||||
"target": "Atlas",
|
||||
"relationship_type": "CREATED",
|
||||
"description": "Acme created Atlas.",
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
def complete_text(self, system: str, user: str) -> str:
|
||||
return "Acme created Atlas. [S1]"
|
||||
|
||||
def embed(self, texts: Sequence[str]) -> list[list[float]]:
|
||||
vectors: list[list[float]] = []
|
||||
for text in texts:
|
||||
normalized = text.casefold()
|
||||
digest = hashlib.sha256(normalized.encode("utf-8")).digest()
|
||||
vectors.append(
|
||||
[
|
||||
float("atlas" in normalized),
|
||||
float("acme" in normalized),
|
||||
digest[0] / 255,
|
||||
]
|
||||
)
|
||||
return vectors
|
||||
|
|
@ -0,0 +1,3 @@
|
|||
{"schema_version":"1.0","external_id":"atlas-overview","corpus":"acceptance-corpus","url":"https://docs.example/atlas","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:00:00Z","body":"<html><head><title>Atlas Overview</title></head><body><main><h1>Atlas</h1><p>Acme Corporation created Atlas. Atlas provides graph retrieval for knowledge applications.</p></main></body></html>","metadata":{"collector":"Web-Access","stored_by":"Data-Store","tags":["product","graph"]}}
|
||||
{"schema_version":"1.0","external_id":"atlas-architecture","corpus":"acceptance-corpus","url":"https://docs.example/atlas/architecture","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:01:00Z","body":"<html><head><title>Atlas Architecture</title></head><body><main><h1>Architecture</h1><p>Acme Corp designed Atlas to combine a knowledge graph with vector retrieval. Atlas depends on PostgreSQL and PgVector.</p></main></body></html>","metadata":{"collector":"Web-Access","stored_by":"Data-Store","description":"Contains commas, \"quotes\", and aliases."}}
|
||||
{"schema_version":"1.0","external_id":"atlas-overview-copy","corpus":"acceptance-corpus","url":"https://mirror.example/atlas","content_type":"text/html","status_code":200,"retrieved_at":"2026-10-13T18:02:00Z","body":"<html><head><title>Atlas Overview</title></head><body><main><h1>Atlas</h1><p>Acme Corporation created Atlas. Atlas provides graph retrieval for knowledge applications.</p></main></body></html>","metadata":{"collector":"Web-Access","stored_by":"Data-Store","duplicate_candidate":true}}
|
||||
|
|
@ -0,0 +1,284 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import datetime, timedelta, timezone
|
||||
from pathlib import Path
|
||||
from uuid import uuid4
|
||||
|
||||
import jwt
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from graphrag_platform.api import create_app
|
||||
from graphrag_platform.auth import Permission
|
||||
from graphrag_platform.config import Settings
|
||||
from graphrag_platform.runtime import StageContext
|
||||
from graphrag_platform.worker import work_once
|
||||
|
||||
from conftest import FakeLLM
|
||||
|
||||
|
||||
def _token(settings: Settings, mask: int) -> str:
|
||||
now = datetime.now(timezone.utc)
|
||||
return jwt.encode(
|
||||
{
|
||||
"iss": settings.internal_jwt_issuer,
|
||||
"aud": settings.internal_jwt_audience,
|
||||
"sub": "dev-test-client",
|
||||
"iat": now,
|
||||
"exp": now + timedelta(minutes=10),
|
||||
"jti": str(uuid4()),
|
||||
"features": {settings.internal_feature_name: mask},
|
||||
},
|
||||
settings.internal_jwt_secret,
|
||||
algorithm="HS256",
|
||||
)
|
||||
|
||||
|
||||
def test_bff_contract_and_worker_execute_durable_job(tmp_path: Path) -> None:
|
||||
source = tmp_path / "source.jsonl"
|
||||
source.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"external_id": "atlas",
|
||||
"corpus": "api-corpus",
|
||||
"url": "https://docs.example/atlas",
|
||||
"retrieved_at": "2026-10-13T18:00:00Z",
|
||||
"body": (
|
||||
"<html><head><title>Atlas</title></head><body><main>"
|
||||
"Acme created Atlas for graph retrieval.</main></body></html>"
|
||||
),
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
database_url=f"sqlite:///{tmp_path / 'api.db'}",
|
||||
auto_create_schema=True,
|
||||
artifact_dir=tmp_path / "artifacts",
|
||||
llm_embedding_dimensions=3,
|
||||
internal_jwt_secret="dev-test-secret-at-least-32-bytes-long",
|
||||
)
|
||||
app = create_app(settings, llm=FakeLLM())
|
||||
full_token = _token(
|
||||
settings,
|
||||
int(Permission.READ | Permission.ADD | Permission.EXECUTE),
|
||||
)
|
||||
|
||||
with TestClient(app) as client:
|
||||
assert client.get("/internal/knowledge-graph/health").status_code == 200
|
||||
assert (
|
||||
client.post(
|
||||
"/internal/knowledge-graph/jobs",
|
||||
json={
|
||||
"corpus": "api-corpus",
|
||||
"from_stage": 1,
|
||||
"through_stage": 8,
|
||||
"input_jsonl": str(source),
|
||||
},
|
||||
).status_code
|
||||
== 401
|
||||
)
|
||||
execute_only = _token(settings, int(Permission.EXECUTE))
|
||||
assert (
|
||||
client.post(
|
||||
"/internal/knowledge-graph/jobs",
|
||||
headers={"Authorization": f"Bearer {execute_only}"},
|
||||
json={
|
||||
"corpus": "api-corpus",
|
||||
"from_stage": 1,
|
||||
"through_stage": 8,
|
||||
"input_jsonl": str(source),
|
||||
},
|
||||
).status_code
|
||||
== 403
|
||||
)
|
||||
queued = client.post(
|
||||
"/internal/knowledge-graph/jobs",
|
||||
headers={"Authorization": f"Bearer {full_token}"},
|
||||
json={
|
||||
"corpus": "api-corpus",
|
||||
"from_stage": 1,
|
||||
"through_stage": 8,
|
||||
"input_jsonl": str(source),
|
||||
},
|
||||
)
|
||||
assert queued.status_code == 202
|
||||
job_id = queued.json()["job_id"]
|
||||
|
||||
assert work_once(StageContext.create(settings, llm=FakeLLM()))
|
||||
|
||||
status_response = client.get(
|
||||
f"/internal/knowledge-graph/jobs/{job_id}",
|
||||
headers={"Authorization": f"Bearer {full_token}"},
|
||||
)
|
||||
assert status_response.status_code == 200
|
||||
assert status_response.json()["status"] == "succeeded"
|
||||
assert len(status_response.json()["stages"]) == 8
|
||||
|
||||
answer = client.post(
|
||||
"/internal/knowledge-graph/answer",
|
||||
headers={"Authorization": f"Bearer {full_token}"},
|
||||
json={"corpus": "api-corpus", "question": "Who created Atlas?"},
|
||||
)
|
||||
assert answer.status_code == 200
|
||||
assert answer.json()["answer"] == "Acme created Atlas. [S1]"
|
||||
|
||||
|
||||
def test_failed_job_can_be_repaired_and_retried(tmp_path: Path) -> None:
|
||||
source = tmp_path / "retry-source.jsonl"
|
||||
source.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "2.0",
|
||||
"external_id": "retry-record",
|
||||
"corpus": "retry-corpus",
|
||||
"url": "https://docs.example/retry",
|
||||
"retrieved_at": "2026-10-13T18:00:00Z",
|
||||
"body": "invalid schema",
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
database_url=f"sqlite:///{tmp_path / 'retry.db'}",
|
||||
auto_create_schema=True,
|
||||
artifact_dir=tmp_path / "artifacts",
|
||||
llm_embedding_dimensions=3,
|
||||
internal_jwt_secret="dev-test-secret-at-least-32-bytes-long",
|
||||
)
|
||||
app = create_app(settings, llm=FakeLLM())
|
||||
token = _token(
|
||||
settings,
|
||||
int(Permission.READ | Permission.ADD | Permission.EXECUTE),
|
||||
)
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
|
||||
with TestClient(app) as client:
|
||||
queued = client.post(
|
||||
"/internal/knowledge-graph/jobs",
|
||||
headers=headers,
|
||||
json={
|
||||
"corpus": "retry-corpus",
|
||||
"from_stage": 1,
|
||||
"through_stage": 1,
|
||||
"input_jsonl": str(source),
|
||||
},
|
||||
)
|
||||
job_id = queued.json()["job_id"]
|
||||
assert work_once(StageContext.create(settings, llm=FakeLLM()))
|
||||
assert (
|
||||
client.get(
|
||||
f"/internal/knowledge-graph/jobs/{job_id}",
|
||||
headers=headers,
|
||||
).json()["status"]
|
||||
== "failed"
|
||||
)
|
||||
|
||||
source.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"external_id": "retry-record",
|
||||
"corpus": "retry-corpus",
|
||||
"url": "https://docs.example/retry",
|
||||
"retrieved_at": "2026-10-13T18:00:00Z",
|
||||
"content_type": "text/plain",
|
||||
"body": "Corrected source content.",
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
retried = client.post(
|
||||
f"/internal/knowledge-graph/jobs/{job_id}/retry",
|
||||
headers=headers,
|
||||
)
|
||||
assert retried.status_code == 202
|
||||
assert work_once(StageContext.create(settings, llm=FakeLLM()))
|
||||
status_response = client.get(
|
||||
f"/internal/knowledge-graph/jobs/{job_id}",
|
||||
headers=headers,
|
||||
)
|
||||
assert status_response.json()["status"] == "succeeded"
|
||||
assert status_response.json()["attempt"] == 2
|
||||
|
||||
|
||||
def test_retry_resumes_after_last_successful_stage(tmp_path: Path) -> None:
|
||||
source = tmp_path / "resume-source.jsonl"
|
||||
source.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"external_id": "resume-record",
|
||||
"corpus": "resume-corpus",
|
||||
"url": "https://docs.example/resume",
|
||||
"retrieved_at": "2026-10-13T18:00:00Z",
|
||||
"content_type": "text/plain",
|
||||
"body": "Acme created Atlas for graph retrieval.",
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
model_file = tmp_path / "model.json"
|
||||
model_file.write_text("[]", encoding="utf-8")
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
database_url=f"sqlite:///{tmp_path / 'resume.db'}",
|
||||
auto_create_schema=True,
|
||||
artifact_dir=tmp_path / "artifacts",
|
||||
llm_embedding_dimensions=3,
|
||||
internal_jwt_secret="dev-test-secret-at-least-32-bytes-long",
|
||||
)
|
||||
app = create_app(settings, llm=FakeLLM())
|
||||
token = _token(
|
||||
settings,
|
||||
int(Permission.READ | Permission.ADD | Permission.EXECUTE),
|
||||
)
|
||||
headers = {"Authorization": f"Bearer {token}"}
|
||||
|
||||
with TestClient(app) as client:
|
||||
queued = client.post(
|
||||
"/internal/knowledge-graph/jobs",
|
||||
headers=headers,
|
||||
json={
|
||||
"corpus": "resume-corpus",
|
||||
"from_stage": 1,
|
||||
"through_stage": 3,
|
||||
"input_jsonl": str(source),
|
||||
"model_file": str(model_file),
|
||||
},
|
||||
)
|
||||
job_id = queued.json()["job_id"]
|
||||
assert work_once(StageContext.create(settings, llm=FakeLLM()))
|
||||
failed = client.get(
|
||||
f"/internal/knowledge-graph/jobs/{job_id}", headers=headers
|
||||
).json()
|
||||
assert failed["status"] == "failed"
|
||||
|
||||
model_file.write_text(
|
||||
json.dumps(settings.model_definition()),
|
||||
encoding="utf-8",
|
||||
)
|
||||
assert (
|
||||
client.post(
|
||||
f"/internal/knowledge-graph/jobs/{job_id}/retry",
|
||||
headers=headers,
|
||||
).status_code
|
||||
== 202
|
||||
)
|
||||
assert work_once(StageContext.create(settings, llm=FakeLLM()))
|
||||
succeeded = client.get(
|
||||
f"/internal/knowledge-graph/jobs/{job_id}", headers=headers
|
||||
).json()
|
||||
|
||||
assert succeeded["status"] == "succeeded"
|
||||
stage_numbers = [row["stage"] for row in succeeded["stages"]]
|
||||
assert stage_numbers.count(1) == 1
|
||||
assert stage_numbers.count(2) == 1
|
||||
assert stage_numbers.count(3) == 2
|
||||
|
|
@ -0,0 +1,69 @@
|
|||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from graphrag_platform.config import Settings
|
||||
from graphrag_platform.models import Community, Edge, Node, SearchItem
|
||||
from graphrag_platform.runtime import StageContext
|
||||
from graphrag_platform.stage_loader import load_stage
|
||||
|
||||
from conftest import FakeLLM
|
||||
|
||||
|
||||
def test_all_ten_stages_build_and_query_graph(tmp_path: Path) -> None:
|
||||
input_file = tmp_path / "documents.jsonl"
|
||||
input_file.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"external_id": "atlas-doc",
|
||||
"corpus": "atlas-corpus",
|
||||
"url": "https://docs.example/atlas",
|
||||
"retrieved_at": "2026-10-13T18:00:00Z",
|
||||
"body": (
|
||||
"<html><head><title>Atlas</title></head><body><main>"
|
||||
"<h1>Atlas</h1><p>Acme created Atlas. Atlas provides graph "
|
||||
"retrieval for knowledge applications.</p></main></body></html>"
|
||||
),
|
||||
"metadata": {"collector": "Web-Access", "stored_by": "Data-Store"},
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
database_url=f"sqlite:///{tmp_path / 'graph.db'}",
|
||||
auto_create_schema=True,
|
||||
artifact_dir=tmp_path / "artifacts",
|
||||
corpus_name="atlas-corpus",
|
||||
chunk_size=300,
|
||||
chunk_overlap=30,
|
||||
retrieval_top_k=5,
|
||||
llm_embedding_dimensions=3,
|
||||
)
|
||||
context = StageContext.create(settings, llm=FakeLLM())
|
||||
|
||||
assert load_stage(1).run(context, input_jsonl=input_file)["normalized"] == 1
|
||||
assert load_stage(2).run(context)["chunks"] == 1
|
||||
assert load_stage(3).run(context)["entity_types"] == 6
|
||||
assert load_stage(4).run(context)["extracted_chunks"] == 1
|
||||
assert load_stage(5).run(context) == {"entities": 2, "relationships": 1}
|
||||
assert load_stage(6).run(context) == {"nodes": 2, "edges": 1}
|
||||
assert load_stage(7).run(context)["communities"] == 1
|
||||
assert load_stage(8).run(context)["indexed_items"] == 5
|
||||
|
||||
retrieval = load_stage(9).run(context, question="Who created Atlas?")
|
||||
response = load_stage(10).run(context, question="Who created Atlas?")
|
||||
|
||||
assert retrieval["context"]
|
||||
assert retrieval["graph_context"][0]["relationship"] == "CREATED"
|
||||
assert retrieval["sources"][0]["url"] == "https://docs.example/atlas"
|
||||
assert response["answer"] == "Acme created Atlas. [S1]"
|
||||
assert response["sources"][0]["citation"] == "S1"
|
||||
with context.storage.session() as session:
|
||||
assert session.scalar(select(func.count()).select_from(Node)) == 2
|
||||
assert session.scalar(select(func.count()).select_from(Edge)) == 1
|
||||
assert session.scalar(select(func.count()).select_from(Community)) == 1
|
||||
assert session.scalar(select(func.count()).select_from(SearchItem)) == 5
|
||||
|
|
@ -0,0 +1,140 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
|
||||
from graphrag_platform.config import Settings
|
||||
from graphrag_platform.models import PipelineRun
|
||||
from graphrag_platform.orchestrator import (
|
||||
RunDeferredError,
|
||||
claim_next_run,
|
||||
execute_run,
|
||||
submit_run,
|
||||
)
|
||||
from graphrag_platform.runtime import StageContext
|
||||
from graphrag_platform.stage_loader import load_stage
|
||||
|
||||
from conftest import FakeLLM
|
||||
|
||||
POSTGRES_URL = os.getenv("TEST_POSTGRES_URL")
|
||||
|
||||
|
||||
@pytest.mark.skipif(
|
||||
not POSTGRES_URL,
|
||||
reason="TEST_POSTGRES_URL is required for the PostgreSQL integration test",
|
||||
)
|
||||
def test_pgvector_index_and_graph_retrieval(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
database_url=POSTGRES_URL,
|
||||
source_database_url=POSTGRES_URL,
|
||||
source_table_name="external_documents",
|
||||
artifact_dir=tmp_path / "artifacts",
|
||||
corpus_name="pgvector-integration",
|
||||
chunk_size=300,
|
||||
chunk_overlap=30,
|
||||
retrieval_top_k=5,
|
||||
llm_embedding_dimensions=3,
|
||||
)
|
||||
monkeypatch.setenv("DATABASE_URL", POSTGRES_URL)
|
||||
from alembic import command
|
||||
from alembic.config import Config
|
||||
|
||||
alembic_config = Config("alembic.ini")
|
||||
command.downgrade(alembic_config, "base")
|
||||
command.upgrade(alembic_config, "head")
|
||||
context = StageContext.create(settings, llm=FakeLLM())
|
||||
context.storage.clear_from_stage(settings.corpus_name, 1)
|
||||
with context.storage.engine.begin() as connection:
|
||||
connection.execute(text("DROP TABLE IF EXISTS external_documents"))
|
||||
connection.execute(
|
||||
text(
|
||||
"""
|
||||
CREATE TABLE external_documents (
|
||||
id bigserial PRIMARY KEY,
|
||||
external_id text NOT NULL UNIQUE,
|
||||
schema_version text NOT NULL,
|
||||
corpus text,
|
||||
url text NOT NULL,
|
||||
body text NOT NULL,
|
||||
content_type text NOT NULL,
|
||||
status_code integer NOT NULL,
|
||||
retrieved_at timestamptz NOT NULL,
|
||||
metadata jsonb NOT NULL,
|
||||
created_at timestamptz NOT NULL DEFAULT now()
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
connection.execute(
|
||||
text(
|
||||
"""
|
||||
INSERT INTO external_documents (
|
||||
external_id, schema_version, corpus, url, body,
|
||||
content_type, status_code, retrieved_at, metadata
|
||||
) VALUES (
|
||||
'atlas-doc', '1.0', 'pgvector-integration',
|
||||
'https://docs.example/atlas',
|
||||
'<html><head><title>Atlas</title></head><body><main><h1>Atlas</h1><p>Acme created Atlas. Atlas provides graph retrieval for knowledge applications.</p></main></body></html>',
|
||||
'text/html', 200, '2026-10-13T18:00:00Z',
|
||||
'{"collector":"Web-Access","stored_by":"Data-Store"}'::jsonb
|
||||
)
|
||||
"""
|
||||
)
|
||||
)
|
||||
|
||||
load_stage(1).run(context)
|
||||
load_stage(2).run(context)
|
||||
load_stage(3).run(context)
|
||||
load_stage(4).run(context)
|
||||
load_stage(5).run(context)
|
||||
load_stage(6).run(context)
|
||||
load_stage(7).run(context)
|
||||
load_stage(8).run(context)
|
||||
result = load_stage(9).run(context, question="Who created Atlas?")
|
||||
|
||||
assert result["context"]
|
||||
assert result["graph_context"][0]["relationship"] == "CREATED"
|
||||
assert result["sources"][0]["url"] == "https://docs.example/atlas"
|
||||
with context.storage.engine.connect() as connection:
|
||||
assert connection.scalar(
|
||||
text("SELECT extversion FROM pg_extension WHERE extname = 'vector'")
|
||||
)
|
||||
index_definition = connection.scalar(
|
||||
text(
|
||||
"SELECT indexdef FROM pg_indexes "
|
||||
"WHERE indexname = 'ix_retrieval_items_embedding_hnsw_3'"
|
||||
)
|
||||
)
|
||||
assert index_definition is not None
|
||||
assert "USING hnsw" in index_definition
|
||||
|
||||
run_id = submit_run(
|
||||
context,
|
||||
from_stage=2,
|
||||
through_stage=2,
|
||||
payload={},
|
||||
requested_by="integration-test",
|
||||
correlation_id="pg-lock-test",
|
||||
permission_mask=31,
|
||||
)
|
||||
with context.storage.engine.connect() as lock_connection:
|
||||
assert lock_connection.scalar(
|
||||
text("SELECT pg_try_advisory_lock(hashtext(:key))"),
|
||||
{"key": "knowledgegraph:pgvector-integration"},
|
||||
)
|
||||
with pytest.raises(RunDeferredError):
|
||||
execute_run(context, run_id)
|
||||
lock_connection.execute(
|
||||
text("SELECT pg_advisory_unlock(hashtext(:key))"),
|
||||
{"key": "knowledgegraph:pgvector-integration"},
|
||||
)
|
||||
with context.storage.session() as session:
|
||||
assert session.get(PipelineRun, run_id).status == "queued"
|
||||
assert claim_next_run(context) == run_id
|
||||
assert execute_run(context, run_id)["2"]["chunks"] == 1
|
||||
|
|
@ -0,0 +1,64 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import time
|
||||
|
||||
import pytest
|
||||
|
||||
from graphrag_platform.config import Settings
|
||||
from graphrag_platform.models import SearchItem
|
||||
from graphrag_platform.runtime import StageContext
|
||||
from graphrag_platform.stage_loader import load_stage
|
||||
|
||||
from conftest import FakeLLM
|
||||
|
||||
POSTGRES_URL = os.getenv("TEST_POSTGRES_URL")
|
||||
RUN_PERFORMANCE_TESTS = os.getenv("RUN_PERFORMANCE_TESTS") == "1"
|
||||
|
||||
|
||||
@pytest.mark.performance
|
||||
@pytest.mark.skipif(
|
||||
not POSTGRES_URL or not RUN_PERFORMANCE_TESTS,
|
||||
reason="Set TEST_POSTGRES_URL and RUN_PERFORMANCE_TESTS=1",
|
||||
)
|
||||
def test_vector_query_latency_at_configured_load() -> None:
|
||||
item_count = int(os.getenv("PERF_VECTOR_ITEM_COUNT", "10000"))
|
||||
maximum_seconds = float(os.getenv("PERF_MAX_QUERY_SECONDS", "2.0"))
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
database_url=POSTGRES_URL,
|
||||
corpus_name="performance-corpus",
|
||||
llm_embedding_dimensions=3,
|
||||
retrieval_top_k=10,
|
||||
)
|
||||
context = StageContext.create(settings, llm=FakeLLM())
|
||||
context.initialize()
|
||||
with context.storage.session() as session:
|
||||
session.query(SearchItem).filter_by(corpus=settings.corpus_name).delete()
|
||||
session.add_all(
|
||||
SearchItem(
|
||||
corpus=settings.corpus_name,
|
||||
item_type="benchmark",
|
||||
item_key=str(index),
|
||||
text=f"Benchmark graph item {index}",
|
||||
source_url=None,
|
||||
metadata_json={},
|
||||
embedding=[
|
||||
float(index % 2),
|
||||
float((index // 2) % 2),
|
||||
(index % 255) / 255,
|
||||
],
|
||||
embedding_model=FakeLLM.embedding_model,
|
||||
)
|
||||
for index in range(item_count)
|
||||
)
|
||||
context.storage.ensure_vector_index(3)
|
||||
|
||||
started = time.perf_counter()
|
||||
result = load_stage(9).run(context, question="Atlas graph retrieval", top_k=10)
|
||||
elapsed = time.perf_counter() - started
|
||||
|
||||
assert len(result["context"]) == 10
|
||||
assert elapsed <= maximum_seconds, (
|
||||
f"vector query took {elapsed:.3f}s; threshold is {maximum_seconds:.3f}s"
|
||||
)
|
||||
|
|
@ -0,0 +1,25 @@
|
|||
from pathlib import Path
|
||||
|
||||
from graphrag_platform.config import Settings
|
||||
|
||||
|
||||
def test_settings_parse_runtime_templates(monkeypatch) -> None:
|
||||
monkeypatch.setenv("ENTITY_TYPES", "Company,Product")
|
||||
monkeypatch.setenv("RELATIONSHIP_TYPES", "CREATED,DEPENDS_ON")
|
||||
monkeypatch.setenv("NODE_TEMPLATE", '{"name":"string"}')
|
||||
|
||||
settings = Settings(_env_file=None)
|
||||
|
||||
assert settings.entity_types == ["Company", "Product"]
|
||||
assert settings.relationship_types == ["CREATED", "DEPENDS_ON"]
|
||||
assert settings.node_template == {"name": "string"}
|
||||
assert settings.database_url.startswith("postgresql+psycopg://")
|
||||
|
||||
|
||||
def test_sqlite_is_available_as_explicit_local_override(tmp_path: Path) -> None:
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
database_url=f"sqlite:///{tmp_path / 'graph.db'}",
|
||||
)
|
||||
|
||||
assert settings.database_url.startswith("sqlite:///")
|
||||
|
|
@ -0,0 +1,66 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
|
||||
from graphrag_platform.config import Settings
|
||||
from graphrag_platform.llm import OpenAICompatibleClient
|
||||
|
||||
|
||||
def test_llm_retries_rate_limit_and_honors_retry_after() -> None:
|
||||
requests = 0
|
||||
delays: list[float] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
nonlocal requests
|
||||
requests += 1
|
||||
if requests == 1:
|
||||
return httpx.Response(
|
||||
429,
|
||||
headers={"Retry-After": "0.25"},
|
||||
json={"error": "rate limited"},
|
||||
request=request,
|
||||
)
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"choices": [{"message": {"content": "answer"}}]},
|
||||
request=request,
|
||||
)
|
||||
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
llm_api_key="test-key",
|
||||
llm_max_retries=2,
|
||||
)
|
||||
client = OpenAICompatibleClient(
|
||||
settings,
|
||||
transport=httpx.MockTransport(handler),
|
||||
sleeper=delays.append,
|
||||
jitter=lambda: 0,
|
||||
)
|
||||
|
||||
assert client.complete_text("system", "user") == "answer"
|
||||
assert requests == 2
|
||||
assert delays == [0.25]
|
||||
|
||||
|
||||
def test_llm_does_not_retry_authentication_failure() -> None:
|
||||
requests = 0
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
nonlocal requests
|
||||
requests += 1
|
||||
return httpx.Response(401, json={"error": "unauthorized"}, request=request)
|
||||
|
||||
client = OpenAICompatibleClient(
|
||||
Settings(_env_file=None, llm_api_key="bad-key", llm_max_retries=3),
|
||||
transport=httpx.MockTransport(handler),
|
||||
sleeper=lambda _: None,
|
||||
)
|
||||
|
||||
try:
|
||||
client.complete_text("system", "user")
|
||||
except httpx.HTTPStatusError:
|
||||
pass
|
||||
else:
|
||||
raise AssertionError("Expected an HTTPStatusError")
|
||||
assert requests == 1
|
||||
|
|
@ -0,0 +1,113 @@
|
|||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import select
|
||||
|
||||
from graphrag_platform.config import Settings
|
||||
from graphrag_platform.models import Chunk, Document, Extraction, ResolvedRelationship
|
||||
from graphrag_platform.runtime import StageContext
|
||||
from graphrag_platform.stage_loader import load_stage
|
||||
|
||||
from conftest import FakeLLM
|
||||
|
||||
|
||||
def test_chunk_overlap_must_be_smaller_than_chunk_size(tmp_path: Path) -> None:
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
database_url=f"sqlite:///{tmp_path / 'graph.db'}",
|
||||
auto_create_schema=True,
|
||||
artifact_dir=tmp_path / "artifacts",
|
||||
chunk_size=200,
|
||||
chunk_overlap=200,
|
||||
)
|
||||
stage = load_stage(2)
|
||||
|
||||
with pytest.raises(ValueError, match="CHUNK_OVERLAP"):
|
||||
stage.chunk_text("word " * 100, size=200, overlap=200)
|
||||
|
||||
|
||||
def test_reconciliation_keeps_cross_chunk_relationship(tmp_path: Path) -> None:
|
||||
settings = Settings(
|
||||
_env_file=None,
|
||||
database_url=f"sqlite:///{tmp_path / 'graph.db'}",
|
||||
auto_create_schema=True,
|
||||
artifact_dir=tmp_path / "artifacts",
|
||||
corpus_name="unit",
|
||||
)
|
||||
context = StageContext.create(settings, llm=FakeLLM())
|
||||
context.initialize()
|
||||
with context.storage.session() as session:
|
||||
document = Document(
|
||||
corpus="unit",
|
||||
url="https://example.test/source",
|
||||
text="Acme created Atlas.",
|
||||
content_hash="document",
|
||||
metadata_json={},
|
||||
)
|
||||
session.add(document)
|
||||
session.flush()
|
||||
first = Chunk(
|
||||
document_id=document.id,
|
||||
corpus="unit",
|
||||
ordinal=0,
|
||||
text="Acme created",
|
||||
content_hash="first",
|
||||
)
|
||||
second = Chunk(
|
||||
document_id=document.id,
|
||||
corpus="unit",
|
||||
ordinal=1,
|
||||
text="Atlas",
|
||||
content_hash="second",
|
||||
)
|
||||
session.add_all([first, second])
|
||||
session.flush()
|
||||
session.add_all(
|
||||
[
|
||||
Extraction(
|
||||
chunk_id=first.id,
|
||||
corpus="unit",
|
||||
model_name="test",
|
||||
payload_json={
|
||||
"entities": [
|
||||
{
|
||||
"canonical_name": "Acme",
|
||||
"entity_type": "Organization",
|
||||
"aliases": [],
|
||||
}
|
||||
],
|
||||
"relationships": [
|
||||
{
|
||||
"source": "Acme",
|
||||
"target": "Atlas",
|
||||
"relationship_type": "CREATED",
|
||||
}
|
||||
],
|
||||
},
|
||||
),
|
||||
Extraction(
|
||||
chunk_id=second.id,
|
||||
corpus="unit",
|
||||
model_name="test",
|
||||
payload_json={
|
||||
"entities": [
|
||||
{
|
||||
"canonical_name": "Atlas",
|
||||
"entity_type": "Product",
|
||||
"aliases": [],
|
||||
}
|
||||
],
|
||||
"relationships": [],
|
||||
},
|
||||
),
|
||||
]
|
||||
)
|
||||
|
||||
result = load_stage(5).run(context)
|
||||
|
||||
assert result == {"entities": 2, "relationships": 1}
|
||||
with context.storage.session() as session:
|
||||
relationship = session.scalar(select(ResolvedRelationship))
|
||||
assert relationship is not None
|
||||
assert relationship.source_key == "acme"
|
||||
assert relationship.target_key == "atlas"
|
||||
|
|
@ -0,0 +1,53 @@
|
|||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from graphrag_platform.sources import JsonLinesSourceReader
|
||||
|
||||
|
||||
def test_jsonl_source_contract_preserves_nested_metadata(tmp_path: Path) -> None:
|
||||
source = tmp_path / "source.jsonl"
|
||||
source.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"external_id": "record-1",
|
||||
"corpus": "test",
|
||||
"url": "https://example.test/article",
|
||||
"body": "<main>Text, with commas and \"quotes\".</main>",
|
||||
"retrieved_at": "2026-10-13T18:00:00Z",
|
||||
"metadata": {"tags": ["one", "two"], "nested": {"value": 1}},
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
records = list(JsonLinesSourceReader(source).records())
|
||||
|
||||
assert records[0].external_id == "record-1"
|
||||
assert records[0].metadata["tags"] == ["one", "two"]
|
||||
assert '"quotes"' in records[0].body
|
||||
|
||||
|
||||
def test_jsonl_source_contract_rejects_unknown_schema(tmp_path: Path) -> None:
|
||||
source = tmp_path / "source.jsonl"
|
||||
source.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"schema_version": "2.0",
|
||||
"external_id": "record-1",
|
||||
"url": "https://example.test/article",
|
||||
"body": "text",
|
||||
"retrieved_at": "2026-10-13T18:00:00Z",
|
||||
}
|
||||
)
|
||||
+ "\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with pytest.raises(ValueError, match="unsupported source schema"):
|
||||
list(JsonLinesSourceReader(source).records())
|
||||
Loading…
Reference in New Issue