129 lines
4.3 KiB
Python
129 lines
4.3 KiB
Python
from __future__ import annotations
|
|
|
|
import argparse
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import trafilatura
|
|
|
|
from graphrag_platform.cli import stage_main
|
|
from graphrag_platform.models import Document, RawDocument
|
|
from graphrag_platform.runtime import StageContext
|
|
from graphrag_platform.sources import SourceRecord, source_reader
|
|
|
|
|
|
def content_hash(text: str) -> str:
|
|
return hashlib.sha256(text.encode("utf-8")).hexdigest()
|
|
|
|
|
|
def extract_document(record: SourceRecord) -> dict[str, object]:
|
|
if "html" not in record.content_type.casefold():
|
|
text = record.body.strip()
|
|
if not text:
|
|
raise ValueError(f"source content is empty for {record.url}")
|
|
return {
|
|
"title": record.metadata.get("title"),
|
|
"author": record.metadata.get("author"),
|
|
"date": record.metadata.get("published_at"),
|
|
"hostname": None,
|
|
"sitename": None,
|
|
"text": text,
|
|
}
|
|
extracted = trafilatura.extract(
|
|
record.body,
|
|
url=str(record.url),
|
|
output_format="json",
|
|
with_metadata=True,
|
|
include_comments=False,
|
|
include_tables=True,
|
|
favor_precision=True,
|
|
)
|
|
if not extracted:
|
|
raise ValueError(f"could not extract article content from {record.url}")
|
|
payload = json.loads(extracted)
|
|
if not str(payload.get("text") or "").strip():
|
|
raise ValueError(f"extracted content is empty for {record.url}")
|
|
return payload
|
|
|
|
|
|
def run(
|
|
context: StageContext,
|
|
*,
|
|
input_jsonl: Path | None = None,
|
|
) -> dict[str, int]:
|
|
context.initialize()
|
|
reader = source_reader(
|
|
input_jsonl=input_jsonl,
|
|
source_database_url=context.settings.source_database_url,
|
|
source_table_name=context.settings.source_table_name,
|
|
)
|
|
normalized = 0
|
|
with context.storage.session() as session:
|
|
for record in reader.records():
|
|
if record.corpus and record.corpus != context.settings.corpus_name:
|
|
raise ValueError(
|
|
f"record {record.external_id} targets corpus {record.corpus}, "
|
|
f"not {context.settings.corpus_name}"
|
|
)
|
|
payload = extract_document(record)
|
|
text = str(payload["text"]).strip()
|
|
metadata = dict(record.metadata)
|
|
metadata.update(
|
|
{
|
|
"external_id": record.external_id,
|
|
"source_schema_version": record.schema_version,
|
|
"retrieved_at": record.retrieved_at.isoformat(),
|
|
"hostname": payload.get("hostname"),
|
|
"sitename": payload.get("sitename"),
|
|
}
|
|
)
|
|
raw = context.storage.upsert(
|
|
session,
|
|
RawDocument,
|
|
{
|
|
"corpus": context.settings.corpus_name,
|
|
"external_id": record.external_id,
|
|
},
|
|
{
|
|
"source_schema_version": record.schema_version,
|
|
"url": str(record.url),
|
|
"status_code": record.status_code,
|
|
"content_type": record.content_type,
|
|
"body": record.body,
|
|
"metadata_json": record.metadata,
|
|
"retrieved_at": record.retrieved_at,
|
|
},
|
|
)
|
|
context.storage.upsert(
|
|
session,
|
|
Document,
|
|
{"raw_document_id": raw.id},
|
|
{
|
|
"corpus": context.settings.corpus_name,
|
|
"url": str(record.url),
|
|
"title": payload.get("title"),
|
|
"author": payload.get("author"),
|
|
"published_at": payload.get("date"),
|
|
"text": text,
|
|
"content_hash": content_hash(text),
|
|
"metadata_json": metadata,
|
|
"is_duplicate": False,
|
|
"canonical_document_id": None,
|
|
},
|
|
)
|
|
normalized += 1
|
|
if normalized == 0:
|
|
raise ValueError("source handoff contained no records")
|
|
return {"normalized": normalized}
|
|
|
|
|
|
def run_from_args(
|
|
context: StageContext, args: argparse.Namespace
|
|
) -> dict[str, int]:
|
|
return run(context, input_jsonl=args.input_jsonl)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
stage_main(1, run_from_args)
|