RAG-POC/tests/integration/test_pgvector_postgres.py

141 lines
4.9 KiB
Python

from __future__ import annotations
import os
from pathlib import Path
import pytest
from sqlalchemy import text
from graphrag_platform.config import Settings
from graphrag_platform.models import PipelineRun
from graphrag_platform.orchestrator import (
RunDeferredError,
claim_next_run,
execute_run,
submit_run,
)
from graphrag_platform.runtime import StageContext
from graphrag_platform.stage_loader import load_stage
from conftest import FakeLLM
POSTGRES_URL = os.getenv("TEST_POSTGRES_URL")
@pytest.mark.skipif(
not POSTGRES_URL,
reason="TEST_POSTGRES_URL is required for the PostgreSQL integration test",
)
def test_pgvector_index_and_graph_retrieval(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
settings = Settings(
_env_file=None,
database_url=POSTGRES_URL,
source_database_url=POSTGRES_URL,
source_table_name="external_documents",
artifact_dir=tmp_path / "artifacts",
corpus_name="pgvector-integration",
chunk_size=300,
chunk_overlap=30,
retrieval_top_k=5,
llm_embedding_dimensions=3,
)
monkeypatch.setenv("DATABASE_URL", POSTGRES_URL)
from alembic import command
from alembic.config import Config
alembic_config = Config("alembic.ini")
command.downgrade(alembic_config, "base")
command.upgrade(alembic_config, "head")
context = StageContext.create(settings, llm=FakeLLM())
context.storage.clear_from_stage(settings.corpus_name, 1)
with context.storage.engine.begin() as connection:
connection.execute(text("DROP TABLE IF EXISTS external_documents"))
connection.execute(
text(
"""
CREATE TABLE external_documents (
id bigserial PRIMARY KEY,
external_id text NOT NULL UNIQUE,
schema_version text NOT NULL,
corpus text,
url text NOT NULL,
body text NOT NULL,
content_type text NOT NULL,
status_code integer NOT NULL,
retrieved_at timestamptz NOT NULL,
metadata jsonb NOT NULL,
created_at timestamptz NOT NULL DEFAULT now()
)
"""
)
)
connection.execute(
text(
"""
INSERT INTO external_documents (
external_id, schema_version, corpus, url, body,
content_type, status_code, retrieved_at, metadata
) VALUES (
'atlas-doc', '1.0', 'pgvector-integration',
'https://docs.example/atlas',
'<html><head><title>Atlas</title></head><body><main><h1>Atlas</h1><p>Acme created Atlas. Atlas provides graph retrieval for knowledge applications.</p></main></body></html>',
'text/html', 200, '2026-10-13T18:00:00Z',
'{"collector":"Web-Access","stored_by":"Data-Store"}'::jsonb
)
"""
)
)
load_stage(1).run(context)
load_stage(2).run(context)
load_stage(3).run(context)
load_stage(4).run(context)
load_stage(5).run(context)
load_stage(6).run(context)
load_stage(7).run(context)
load_stage(8).run(context)
result = load_stage(9).run(context, question="Who created Atlas?")
assert result["context"]
assert result["graph_context"][0]["relationship"] == "CREATED"
assert result["sources"][0]["url"] == "https://docs.example/atlas"
with context.storage.engine.connect() as connection:
assert connection.scalar(
text("SELECT extversion FROM pg_extension WHERE extname = 'vector'")
)
index_definition = connection.scalar(
text(
"SELECT indexdef FROM pg_indexes "
"WHERE indexname = 'ix_retrieval_items_embedding_hnsw_3'"
)
)
assert index_definition is not None
assert "USING hnsw" in index_definition
run_id = submit_run(
context,
from_stage=2,
through_stage=2,
payload={},
requested_by="integration-test",
correlation_id="pg-lock-test",
permission_mask=31,
)
with context.storage.engine.connect() as lock_connection:
assert lock_connection.scalar(
text("SELECT pg_try_advisory_lock(hashtext(:key))"),
{"key": "knowledgegraph:pgvector-integration"},
)
with pytest.raises(RunDeferredError):
execute_run(context, run_id)
lock_connection.execute(
text("SELECT pg_advisory_unlock(hashtext(:key))"),
{"key": "knowledgegraph:pgvector-integration"},
)
with context.storage.session() as session:
assert session.get(PipelineRun, run_id).status == "queued"
assert claim_next_run(context) == run_id
assert execute_run(context, run_id)["2"]["chunks"] == 1