RAG-POC/src/graphrag_platform/models.py

281 lines
11 KiB
Python

from __future__ import annotations
from datetime import datetime, timezone
from typing import Any
from uuid import uuid4
from sqlalchemy import (
DateTime,
Float,
ForeignKey,
Index,
Integer,
JSON,
String,
Text,
UniqueConstraint,
)
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
from pgvector.sqlalchemy import Vector
def utc_now() -> datetime:
return datetime.now(timezone.utc)
class Base(DeclarativeBase):
pass
class RawDocument(Base):
__tablename__ = "raw_documents"
__table_args__ = (
UniqueConstraint("corpus", "external_id", name="uq_raw_corpus_external_id"),
)
id: Mapped[int] = mapped_column(primary_key=True)
corpus: Mapped[str] = mapped_column(String(200), index=True)
external_id: Mapped[str] = mapped_column(String(500))
source_schema_version: Mapped[str] = mapped_column(String(20))
url: Mapped[str] = mapped_column(Text)
content_type: Mapped[str | None] = mapped_column(String(200))
status_code: Mapped[int | None] = mapped_column(Integer)
body: Mapped[str] = mapped_column(Text)
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
collected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
class Document(Base):
__tablename__ = "documents"
__table_args__ = (
UniqueConstraint("corpus", "url", name="uq_document_corpus_url"),
UniqueConstraint("raw_document_id", name="uq_document_raw_document"),
)
id: Mapped[int] = mapped_column(primary_key=True)
raw_document_id: Mapped[int | None] = mapped_column(
ForeignKey("raw_documents.id", ondelete="SET NULL")
)
corpus: Mapped[str] = mapped_column(String(200), index=True)
url: Mapped[str] = mapped_column(Text)
title: Mapped[str | None] = mapped_column(Text)
author: Mapped[str | None] = mapped_column(Text)
published_at: Mapped[str | None] = mapped_column(String(100))
text: Mapped[str] = mapped_column(Text)
content_hash: Mapped[str] = mapped_column(String(64), index=True)
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
is_duplicate: Mapped[bool] = mapped_column(default=False)
canonical_document_id: Mapped[int | None] = mapped_column(
ForeignKey("documents.id", ondelete="SET NULL")
)
class Chunk(Base):
__tablename__ = "chunks"
__table_args__ = (UniqueConstraint("document_id", "ordinal", name="uq_chunk_ordinal"),)
id: Mapped[int] = mapped_column(primary_key=True)
document_id: Mapped[int] = mapped_column(
ForeignKey("documents.id", ondelete="CASCADE"), index=True
)
corpus: Mapped[str] = mapped_column(String(200), index=True)
ordinal: Mapped[int] = mapped_column(Integer)
text: Mapped[str] = mapped_column(Text)
content_hash: Mapped[str] = mapped_column(String(64), index=True)
class KnowledgeModel(Base):
__tablename__ = "knowledge_models"
corpus: Mapped[str] = mapped_column(String(200), primary_key=True)
definition_json: Mapped[dict[str, Any]] = mapped_column(JSON)
updated_at: Mapped[datetime] = mapped_column(
DateTime(timezone=True), default=utc_now, onupdate=utc_now
)
class Extraction(Base):
__tablename__ = "extractions"
__table_args__ = (UniqueConstraint("chunk_id", name="uq_extraction_chunk"),)
id: Mapped[int] = mapped_column(primary_key=True)
chunk_id: Mapped[int] = mapped_column(
ForeignKey("chunks.id", ondelete="CASCADE"), index=True
)
corpus: Mapped[str] = mapped_column(String(200), index=True)
payload_json: Mapped[dict[str, Any]] = mapped_column(JSON)
model_name: Mapped[str] = mapped_column(String(200))
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
class ResolvedEntity(Base):
__tablename__ = "resolved_entities"
__table_args__ = (
UniqueConstraint("corpus", "canonical_key", name="uq_resolved_entity_key"),
)
id: Mapped[int] = mapped_column(primary_key=True)
corpus: Mapped[str] = mapped_column(String(200), index=True)
canonical_key: Mapped[str] = mapped_column(String(500))
canonical_name: Mapped[str] = mapped_column(Text)
entity_type: Mapped[str] = mapped_column(String(100))
description: Mapped[str] = mapped_column(Text, default="")
aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list)
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
class ResolvedRelationship(Base):
__tablename__ = "resolved_relationships"
__table_args__ = (
UniqueConstraint(
"corpus",
"source_key",
"target_key",
"relationship_type",
name="uq_resolved_relationship",
),
)
id: Mapped[int] = mapped_column(primary_key=True)
corpus: Mapped[str] = mapped_column(String(200), index=True)
source_key: Mapped[str] = mapped_column(String(500))
target_key: Mapped[str] = mapped_column(String(500))
relationship_type: Mapped[str] = mapped_column(String(100))
description: Mapped[str] = mapped_column(Text, default="")
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
class Node(Base):
__tablename__ = "nodes"
__table_args__ = (
UniqueConstraint("corpus", "canonical_key", name="uq_node_canonical_key"),
)
id: Mapped[int] = mapped_column(primary_key=True)
corpus: Mapped[str] = mapped_column(String(200), index=True)
canonical_key: Mapped[str] = mapped_column(String(500))
canonical_name: Mapped[str] = mapped_column(Text)
entity_type: Mapped[str] = mapped_column(String(100), index=True)
description: Mapped[str] = mapped_column(Text, default="")
aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list)
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
class Edge(Base):
__tablename__ = "edges"
__table_args__ = (
UniqueConstraint(
"corpus",
"source_node_id",
"target_node_id",
"relationship_type",
name="uq_edge_identity",
),
)
id: Mapped[int] = mapped_column(primary_key=True)
corpus: Mapped[str] = mapped_column(String(200), index=True)
source_node_id: Mapped[int] = mapped_column(
ForeignKey("nodes.id", ondelete="CASCADE"), index=True
)
target_node_id: Mapped[int] = mapped_column(
ForeignKey("nodes.id", ondelete="CASCADE"), index=True
)
relationship_type: Mapped[str] = mapped_column(String(100), index=True)
description: Mapped[str] = mapped_column(Text, default="")
weight: Mapped[float] = mapped_column(Float, default=1.0)
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
class Community(Base):
__tablename__ = "communities"
__table_args__ = (UniqueConstraint("corpus", "community_key", name="uq_community_key"),)
id: Mapped[int] = mapped_column(primary_key=True)
corpus: Mapped[str] = mapped_column(String(200), index=True)
community_key: Mapped[str] = mapped_column(String(100))
title: Mapped[str] = mapped_column(Text)
summary: Mapped[str] = mapped_column(Text)
node_ids_json: Mapped[list[int]] = mapped_column(JSON)
class SearchItem(Base):
__tablename__ = "retrieval_items"
__table_args__ = (
UniqueConstraint("corpus", "item_type", "item_key", name="uq_search_item"),
Index("ix_retrieval_items_corpus_type", "corpus", "item_type"),
Index(
"ix_retrieval_items_corpus_model",
"corpus",
"embedding_model",
),
)
id: Mapped[int] = mapped_column(primary_key=True)
corpus: Mapped[str] = mapped_column(String(200), index=True)
item_type: Mapped[str] = mapped_column(String(50))
item_key: Mapped[str] = mapped_column(String(100))
text: Mapped[str] = mapped_column(Text)
source_url: Mapped[str | None] = mapped_column(Text)
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
embedding: Mapped[list[float]] = mapped_column(
Vector().with_variant(JSON(), "sqlite")
)
embedding_model: Mapped[str] = mapped_column(String(200))
class PipelineRun(Base):
__tablename__ = "pipeline_runs"
__table_args__ = (
Index("ix_pipeline_runs_status_created", "status", "created_at"),
Index("ix_pipeline_runs_corpus_created", "corpus", "created_at"),
)
id: Mapped[str] = mapped_column(
String(36), primary_key=True, default=lambda: str(uuid4())
)
corpus: Mapped[str] = mapped_column(String(200), index=True)
status: Mapped[str] = mapped_column(String(30), index=True, default="queued")
from_stage: Mapped[int] = mapped_column(Integer)
through_stage: Mapped[int] = mapped_column(Integer)
payload_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
configuration_hash: Mapped[str] = mapped_column(String(64))
requested_by: Mapped[str] = mapped_column(String(500))
correlation_id: Mapped[str] = mapped_column(String(200), index=True)
feature_name: Mapped[str] = mapped_column(String(200), default="KnowledgeGraph")
permission_mask: Mapped[int] = mapped_column(Integer)
attempt: Mapped[int] = mapped_column(Integer, default=0)
output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
error_message: Mapped[str | None] = mapped_column(Text)
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
class StageRun(Base):
__tablename__ = "stage_runs"
__table_args__ = (
UniqueConstraint(
"pipeline_run_id",
"stage_number",
"attempt",
name="uq_stage_run_attempt",
),
Index("ix_stage_runs_pipeline_stage", "pipeline_run_id", "stage_number"),
)
id: Mapped[int] = mapped_column(primary_key=True)
pipeline_run_id: Mapped[str] = mapped_column(
ForeignKey("pipeline_runs.id", ondelete="CASCADE"), index=True
)
stage_number: Mapped[int] = mapped_column(Integer)
stage_name: Mapped[str] = mapped_column(String(200))
status: Mapped[str] = mapped_column(String(30), default="running")
attempt: Mapped[int] = mapped_column(Integer)
input_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
error_message: Mapped[str | None] = mapped_column(Text)
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))