98 lines
3.6 KiB
Python
98 lines
3.6 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import json
|
|
from typing import Any
|
|
|
|
from sqlalchemy import select
|
|
|
|
from graphrag_platform.cli import stage_main
|
|
from graphrag_platform.models import Chunk, Extraction, KnowledgeModel
|
|
from graphrag_platform.runtime import StageContext
|
|
|
|
|
|
def validate_extraction(
|
|
payload: dict[str, Any], definition: dict[str, Any]
|
|
) -> None:
|
|
entities = payload.get("entities")
|
|
relationships = payload.get("relationships")
|
|
if not isinstance(entities, list) or not isinstance(relationships, list):
|
|
raise ValueError("extraction requires entities and relationships arrays")
|
|
entity_types = set(definition["entity_types"])
|
|
relationship_types = set(definition["relationship_types"])
|
|
for entity in entities:
|
|
if not isinstance(entity, dict):
|
|
raise ValueError("each entity must be an object")
|
|
if not str(entity.get("canonical_name", "")).strip():
|
|
raise ValueError("entity canonical_name must not be empty")
|
|
if entity.get("entity_type") not in entity_types:
|
|
raise ValueError(f"unsupported entity type: {entity.get('entity_type')}")
|
|
if not isinstance(entity.get("aliases", []), list):
|
|
raise ValueError("entity aliases must be an array")
|
|
for relationship in relationships:
|
|
if not isinstance(relationship, dict):
|
|
raise ValueError("each relationship must be an object")
|
|
if not str(relationship.get("source", "")).strip() or not str(
|
|
relationship.get("target", "")
|
|
).strip():
|
|
raise ValueError("relationship source and target must not be empty")
|
|
if relationship.get("relationship_type") not in relationship_types:
|
|
raise ValueError(
|
|
"unsupported relationship type: "
|
|
f"{relationship.get('relationship_type')}"
|
|
)
|
|
|
|
|
|
def run(context: StageContext) -> dict[str, int]:
|
|
context.initialize()
|
|
with context.storage.session() as session:
|
|
model = session.get(KnowledgeModel, context.settings.corpus_name)
|
|
chunks = session.scalars(
|
|
select(Chunk).where(Chunk.corpus == context.settings.corpus_name)
|
|
).all()
|
|
if model is None or not chunks:
|
|
raise ValueError("stage 4 requires a knowledge model and cleaned chunks")
|
|
definition = dict(model.definition_json)
|
|
chunk_inputs = [(chunk.id, chunk.text) for chunk in chunks]
|
|
|
|
system = (
|
|
"You extract a source-grounded knowledge graph. Return one JSON object "
|
|
'with arrays named "entities" and "relationships". Never infer unsupported facts.'
|
|
)
|
|
extracted_count = 0
|
|
for chunk_id, chunk_text in chunk_inputs:
|
|
prompt = json.dumps(
|
|
{
|
|
"knowledge_model": definition,
|
|
"source_chunk_id": chunk_id,
|
|
"text": chunk_text,
|
|
},
|
|
ensure_ascii=False,
|
|
)
|
|
payload = context.llm.complete_json(system, prompt)
|
|
validate_extraction(payload, definition)
|
|
with context.storage.session() as session:
|
|
context.storage.upsert(
|
|
session,
|
|
Extraction,
|
|
{"chunk_id": chunk_id},
|
|
{
|
|
"corpus": context.settings.corpus_name,
|
|
"payload_json": payload,
|
|
"model_name": context.llm.chat_model,
|
|
},
|
|
)
|
|
extracted_count += 1
|
|
return {"extracted_chunks": extracted_count}
|
|
|
|
|
|
def run_from_args(
|
|
context: StageContext, args: argparse.Namespace
|
|
) -> dict[str, int]:
|
|
del args
|
|
return run(context)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
stage_main(4, run_from_args)
|