from __future__ import annotations import argparse import hashlib import json from pathlib import Path import trafilatura from graphrag_platform.cli import stage_main from graphrag_platform.models import Document, RawDocument from graphrag_platform.runtime import StageContext from graphrag_platform.sources import SourceRecord, source_reader def content_hash(text: str) -> str: return hashlib.sha256(text.encode("utf-8")).hexdigest() def extract_document(record: SourceRecord) -> dict[str, object]: if "html" not in record.content_type.casefold(): text = record.body.strip() if not text: raise ValueError(f"source content is empty for {record.url}") return { "title": record.metadata.get("title"), "author": record.metadata.get("author"), "date": record.metadata.get("published_at"), "hostname": None, "sitename": None, "text": text, } extracted = trafilatura.extract( record.body, url=str(record.url), output_format="json", with_metadata=True, include_comments=False, include_tables=True, favor_precision=True, ) if not extracted: raise ValueError(f"could not extract article content from {record.url}") payload = json.loads(extracted) if not str(payload.get("text") or "").strip(): raise ValueError(f"extracted content is empty for {record.url}") return payload def run( context: StageContext, *, input_jsonl: Path | None = None, ) -> dict[str, int]: context.initialize() reader = source_reader( input_jsonl=input_jsonl, source_database_url=context.settings.source_database_url, source_table_name=context.settings.source_table_name, ) normalized = 0 with context.storage.session() as session: for record in reader.records(): if record.corpus and record.corpus != context.settings.corpus_name: raise ValueError( f"record {record.external_id} targets corpus {record.corpus}, " f"not {context.settings.corpus_name}" ) payload = extract_document(record) text = str(payload["text"]).strip() metadata = dict(record.metadata) metadata.update( { "external_id": record.external_id, "source_schema_version": record.schema_version, "retrieved_at": record.retrieved_at.isoformat(), "hostname": payload.get("hostname"), "sitename": payload.get("sitename"), } ) raw = context.storage.upsert( session, RawDocument, { "corpus": context.settings.corpus_name, "external_id": record.external_id, }, { "source_schema_version": record.schema_version, "url": str(record.url), "status_code": record.status_code, "content_type": record.content_type, "body": record.body, "metadata_json": record.metadata, "retrieved_at": record.retrieved_at, }, ) context.storage.upsert( session, Document, {"raw_document_id": raw.id}, { "corpus": context.settings.corpus_name, "url": str(record.url), "title": payload.get("title"), "author": payload.get("author"), "published_at": payload.get("date"), "text": text, "content_hash": content_hash(text), "metadata_json": metadata, "is_duplicate": False, "canonical_document_id": None, }, ) normalized += 1 if normalized == 0: raise ValueError("source handoff contained no records") return {"normalized": normalized} def run_from_args( context: StageContext, args: argparse.Namespace ) -> dict[str, int]: return run(context, input_jsonl=args.input_jsonl) if __name__ == "__main__": stage_main(1, run_from_args)