from __future__ import annotations from datetime import datetime, timezone from typing import Any from uuid import uuid4 from sqlalchemy import ( DateTime, Float, ForeignKey, Index, Integer, JSON, String, Text, UniqueConstraint, ) from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column from pgvector.sqlalchemy import Vector def utc_now() -> datetime: return datetime.now(timezone.utc) class Base(DeclarativeBase): pass class RawDocument(Base): __tablename__ = "raw_documents" __table_args__ = ( UniqueConstraint("corpus", "external_id", name="uq_raw_corpus_external_id"), ) id: Mapped[int] = mapped_column(primary_key=True) corpus: Mapped[str] = mapped_column(String(200), index=True) external_id: Mapped[str] = mapped_column(String(500)) source_schema_version: Mapped[str] = mapped_column(String(20)) url: Mapped[str] = mapped_column(Text) content_type: Mapped[str | None] = mapped_column(String(200)) status_code: Mapped[int | None] = mapped_column(Integer) body: Mapped[str] = mapped_column(Text) metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True)) collected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now) class Document(Base): __tablename__ = "documents" __table_args__ = ( UniqueConstraint("corpus", "url", name="uq_document_corpus_url"), UniqueConstraint("raw_document_id", name="uq_document_raw_document"), ) id: Mapped[int] = mapped_column(primary_key=True) raw_document_id: Mapped[int | None] = mapped_column( ForeignKey("raw_documents.id", ondelete="SET NULL") ) corpus: Mapped[str] = mapped_column(String(200), index=True) url: Mapped[str] = mapped_column(Text) title: Mapped[str | None] = mapped_column(Text) author: Mapped[str | None] = mapped_column(Text) published_at: Mapped[str | None] = mapped_column(String(100)) text: Mapped[str] = mapped_column(Text) content_hash: Mapped[str] = mapped_column(String(64), index=True) metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) is_duplicate: Mapped[bool] = mapped_column(default=False) canonical_document_id: Mapped[int | None] = mapped_column( ForeignKey("documents.id", ondelete="SET NULL") ) class Chunk(Base): __tablename__ = "chunks" __table_args__ = (UniqueConstraint("document_id", "ordinal", name="uq_chunk_ordinal"),) id: Mapped[int] = mapped_column(primary_key=True) document_id: Mapped[int] = mapped_column( ForeignKey("documents.id", ondelete="CASCADE"), index=True ) corpus: Mapped[str] = mapped_column(String(200), index=True) ordinal: Mapped[int] = mapped_column(Integer) text: Mapped[str] = mapped_column(Text) content_hash: Mapped[str] = mapped_column(String(64), index=True) class KnowledgeModel(Base): __tablename__ = "knowledge_models" corpus: Mapped[str] = mapped_column(String(200), primary_key=True) definition_json: Mapped[dict[str, Any]] = mapped_column(JSON) updated_at: Mapped[datetime] = mapped_column( DateTime(timezone=True), default=utc_now, onupdate=utc_now ) class Extraction(Base): __tablename__ = "extractions" __table_args__ = (UniqueConstraint("chunk_id", name="uq_extraction_chunk"),) id: Mapped[int] = mapped_column(primary_key=True) chunk_id: Mapped[int] = mapped_column( ForeignKey("chunks.id", ondelete="CASCADE"), index=True ) corpus: Mapped[str] = mapped_column(String(200), index=True) payload_json: Mapped[dict[str, Any]] = mapped_column(JSON) model_name: Mapped[str] = mapped_column(String(200)) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now) class ResolvedEntity(Base): __tablename__ = "resolved_entities" __table_args__ = ( UniqueConstraint("corpus", "canonical_key", name="uq_resolved_entity_key"), ) id: Mapped[int] = mapped_column(primary_key=True) corpus: Mapped[str] = mapped_column(String(200), index=True) canonical_key: Mapped[str] = mapped_column(String(500)) canonical_name: Mapped[str] = mapped_column(Text) entity_type: Mapped[str] = mapped_column(String(100)) description: Mapped[str] = mapped_column(Text, default="") aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list) source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list) class ResolvedRelationship(Base): __tablename__ = "resolved_relationships" __table_args__ = ( UniqueConstraint( "corpus", "source_key", "target_key", "relationship_type", name="uq_resolved_relationship", ), ) id: Mapped[int] = mapped_column(primary_key=True) corpus: Mapped[str] = mapped_column(String(200), index=True) source_key: Mapped[str] = mapped_column(String(500)) target_key: Mapped[str] = mapped_column(String(500)) relationship_type: Mapped[str] = mapped_column(String(100)) description: Mapped[str] = mapped_column(Text, default="") source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list) class Node(Base): __tablename__ = "nodes" __table_args__ = ( UniqueConstraint("corpus", "canonical_key", name="uq_node_canonical_key"), ) id: Mapped[int] = mapped_column(primary_key=True) corpus: Mapped[str] = mapped_column(String(200), index=True) canonical_key: Mapped[str] = mapped_column(String(500)) canonical_name: Mapped[str] = mapped_column(Text) entity_type: Mapped[str] = mapped_column(String(100), index=True) description: Mapped[str] = mapped_column(Text, default="") aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list) source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list) class Edge(Base): __tablename__ = "edges" __table_args__ = ( UniqueConstraint( "corpus", "source_node_id", "target_node_id", "relationship_type", name="uq_edge_identity", ), ) id: Mapped[int] = mapped_column(primary_key=True) corpus: Mapped[str] = mapped_column(String(200), index=True) source_node_id: Mapped[int] = mapped_column( ForeignKey("nodes.id", ondelete="CASCADE"), index=True ) target_node_id: Mapped[int] = mapped_column( ForeignKey("nodes.id", ondelete="CASCADE"), index=True ) relationship_type: Mapped[str] = mapped_column(String(100), index=True) description: Mapped[str] = mapped_column(Text, default="") weight: Mapped[float] = mapped_column(Float, default=1.0) source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list) class Community(Base): __tablename__ = "communities" __table_args__ = (UniqueConstraint("corpus", "community_key", name="uq_community_key"),) id: Mapped[int] = mapped_column(primary_key=True) corpus: Mapped[str] = mapped_column(String(200), index=True) community_key: Mapped[str] = mapped_column(String(100)) title: Mapped[str] = mapped_column(Text) summary: Mapped[str] = mapped_column(Text) node_ids_json: Mapped[list[int]] = mapped_column(JSON) class SearchItem(Base): __tablename__ = "retrieval_items" __table_args__ = ( UniqueConstraint("corpus", "item_type", "item_key", name="uq_search_item"), Index("ix_retrieval_items_corpus_type", "corpus", "item_type"), Index( "ix_retrieval_items_corpus_model", "corpus", "embedding_model", ), ) id: Mapped[int] = mapped_column(primary_key=True) corpus: Mapped[str] = mapped_column(String(200), index=True) item_type: Mapped[str] = mapped_column(String(50)) item_key: Mapped[str] = mapped_column(String(100)) text: Mapped[str] = mapped_column(Text) source_url: Mapped[str | None] = mapped_column(Text) metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) embedding: Mapped[list[float]] = mapped_column( Vector().with_variant(JSON(), "sqlite") ) embedding_model: Mapped[str] = mapped_column(String(200)) class PipelineRun(Base): __tablename__ = "pipeline_runs" __table_args__ = ( Index("ix_pipeline_runs_status_created", "status", "created_at"), Index("ix_pipeline_runs_corpus_created", "corpus", "created_at"), ) id: Mapped[str] = mapped_column( String(36), primary_key=True, default=lambda: str(uuid4()) ) corpus: Mapped[str] = mapped_column(String(200), index=True) status: Mapped[str] = mapped_column(String(30), index=True, default="queued") from_stage: Mapped[int] = mapped_column(Integer) through_stage: Mapped[int] = mapped_column(Integer) payload_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) configuration_hash: Mapped[str] = mapped_column(String(64)) requested_by: Mapped[str] = mapped_column(String(500)) correlation_id: Mapped[str] = mapped_column(String(200), index=True) feature_name: Mapped[str] = mapped_column(String(200), default="KnowledgeGraph") permission_mask: Mapped[int] = mapped_column(Integer) attempt: Mapped[int] = mapped_column(Integer, default=0) output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) error_message: Mapped[str | None] = mapped_column(Text) created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now) started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) class StageRun(Base): __tablename__ = "stage_runs" __table_args__ = ( UniqueConstraint( "pipeline_run_id", "stage_number", "attempt", name="uq_stage_run_attempt", ), Index("ix_stage_runs_pipeline_stage", "pipeline_run_id", "stage_number"), ) id: Mapped[int] = mapped_column(primary_key=True) pipeline_run_id: Mapped[str] = mapped_column( ForeignKey("pipeline_runs.id", ondelete="CASCADE"), index=True ) stage_number: Mapped[int] = mapped_column(Integer) stage_name: Mapped[str] = mapped_column(String(200)) status: Mapped[str] = mapped_column(String(30), default="running") attempt: Mapped[int] = mapped_column(Integer) input_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict) error_message: Mapped[str | None] = mapped_column(Text) started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now) finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))