RAG-POC/src/01_normalize_content.py

129 lines
4.3 KiB
Python

from __future__ import annotations
import argparse
import hashlib
import json
from pathlib import Path
import trafilatura
from graphrag_platform.cli import stage_main
from graphrag_platform.models import Document, RawDocument
from graphrag_platform.runtime import StageContext
from graphrag_platform.sources import SourceRecord, source_reader
def content_hash(text: str) -> str:
return hashlib.sha256(text.encode("utf-8")).hexdigest()
def extract_document(record: SourceRecord) -> dict[str, object]:
if "html" not in record.content_type.casefold():
text = record.body.strip()
if not text:
raise ValueError(f"source content is empty for {record.url}")
return {
"title": record.metadata.get("title"),
"author": record.metadata.get("author"),
"date": record.metadata.get("published_at"),
"hostname": None,
"sitename": None,
"text": text,
}
extracted = trafilatura.extract(
record.body,
url=str(record.url),
output_format="json",
with_metadata=True,
include_comments=False,
include_tables=True,
favor_precision=True,
)
if not extracted:
raise ValueError(f"could not extract article content from {record.url}")
payload = json.loads(extracted)
if not str(payload.get("text") or "").strip():
raise ValueError(f"extracted content is empty for {record.url}")
return payload
def run(
context: StageContext,
*,
input_jsonl: Path | None = None,
) -> dict[str, int]:
context.initialize()
reader = source_reader(
input_jsonl=input_jsonl,
source_database_url=context.settings.source_database_url,
source_table_name=context.settings.source_table_name,
)
normalized = 0
with context.storage.session() as session:
for record in reader.records():
if record.corpus and record.corpus != context.settings.corpus_name:
raise ValueError(
f"record {record.external_id} targets corpus {record.corpus}, "
f"not {context.settings.corpus_name}"
)
payload = extract_document(record)
text = str(payload["text"]).strip()
metadata = dict(record.metadata)
metadata.update(
{
"external_id": record.external_id,
"source_schema_version": record.schema_version,
"retrieved_at": record.retrieved_at.isoformat(),
"hostname": payload.get("hostname"),
"sitename": payload.get("sitename"),
}
)
raw = context.storage.upsert(
session,
RawDocument,
{
"corpus": context.settings.corpus_name,
"external_id": record.external_id,
},
{
"source_schema_version": record.schema_version,
"url": str(record.url),
"status_code": record.status_code,
"content_type": record.content_type,
"body": record.body,
"metadata_json": record.metadata,
"retrieved_at": record.retrieved_at,
},
)
context.storage.upsert(
session,
Document,
{"raw_document_id": raw.id},
{
"corpus": context.settings.corpus_name,
"url": str(record.url),
"title": payload.get("title"),
"author": payload.get("author"),
"published_at": payload.get("date"),
"text": text,
"content_hash": content_hash(text),
"metadata_json": metadata,
"is_duplicate": False,
"canonical_document_id": None,
},
)
normalized += 1
if normalized == 0:
raise ValueError("source handoff contained no records")
return {"normalized": normalized}
def run_from_args(
context: StageContext, args: argparse.Namespace
) -> dict[str, int]:
return run(context, input_jsonl=args.input_jsonl)
if __name__ == "__main__":
stage_main(1, run_from_args)