RAG-POC/src/04_extract_knowledge.py

98 lines
3.6 KiB
Python

from __future__ import annotations
import argparse
import json
from typing import Any
from sqlalchemy import select
from graphrag_platform.cli import stage_main
from graphrag_platform.models import Chunk, Extraction, KnowledgeModel
from graphrag_platform.runtime import StageContext
def validate_extraction(
payload: dict[str, Any], definition: dict[str, Any]
) -> None:
entities = payload.get("entities")
relationships = payload.get("relationships")
if not isinstance(entities, list) or not isinstance(relationships, list):
raise ValueError("extraction requires entities and relationships arrays")
entity_types = set(definition["entity_types"])
relationship_types = set(definition["relationship_types"])
for entity in entities:
if not isinstance(entity, dict):
raise ValueError("each entity must be an object")
if not str(entity.get("canonical_name", "")).strip():
raise ValueError("entity canonical_name must not be empty")
if entity.get("entity_type") not in entity_types:
raise ValueError(f"unsupported entity type: {entity.get('entity_type')}")
if not isinstance(entity.get("aliases", []), list):
raise ValueError("entity aliases must be an array")
for relationship in relationships:
if not isinstance(relationship, dict):
raise ValueError("each relationship must be an object")
if not str(relationship.get("source", "")).strip() or not str(
relationship.get("target", "")
).strip():
raise ValueError("relationship source and target must not be empty")
if relationship.get("relationship_type") not in relationship_types:
raise ValueError(
"unsupported relationship type: "
f"{relationship.get('relationship_type')}"
)
def run(context: StageContext) -> dict[str, int]:
context.initialize()
with context.storage.session() as session:
model = session.get(KnowledgeModel, context.settings.corpus_name)
chunks = session.scalars(
select(Chunk).where(Chunk.corpus == context.settings.corpus_name)
).all()
if model is None or not chunks:
raise ValueError("stage 4 requires a knowledge model and cleaned chunks")
definition = dict(model.definition_json)
chunk_inputs = [(chunk.id, chunk.text) for chunk in chunks]
system = (
"You extract a source-grounded knowledge graph. Return one JSON object "
'with arrays named "entities" and "relationships". Never infer unsupported facts.'
)
extracted_count = 0
for chunk_id, chunk_text in chunk_inputs:
prompt = json.dumps(
{
"knowledge_model": definition,
"source_chunk_id": chunk_id,
"text": chunk_text,
},
ensure_ascii=False,
)
payload = context.llm.complete_json(system, prompt)
validate_extraction(payload, definition)
with context.storage.session() as session:
context.storage.upsert(
session,
Extraction,
{"chunk_id": chunk_id},
{
"corpus": context.settings.corpus_name,
"payload_json": payload,
"model_name": context.llm.chat_model,
},
)
extracted_count += 1
return {"extracted_chunks": extracted_count}
def run_from_args(
context: StageContext, args: argparse.Namespace
) -> dict[str, int]:
del args
return run(context)
if __name__ == "__main__":
stage_main(4, run_from_args)