281 lines
11 KiB
Python
281 lines
11 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
from typing import Any
|
|
from uuid import uuid4
|
|
|
|
from sqlalchemy import (
|
|
DateTime,
|
|
Float,
|
|
ForeignKey,
|
|
Index,
|
|
Integer,
|
|
JSON,
|
|
String,
|
|
Text,
|
|
UniqueConstraint,
|
|
)
|
|
from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
|
|
from pgvector.sqlalchemy import Vector
|
|
|
|
|
|
def utc_now() -> datetime:
|
|
return datetime.now(timezone.utc)
|
|
|
|
|
|
class Base(DeclarativeBase):
|
|
pass
|
|
|
|
|
|
class RawDocument(Base):
|
|
__tablename__ = "raw_documents"
|
|
__table_args__ = (
|
|
UniqueConstraint("corpus", "external_id", name="uq_raw_corpus_external_id"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
|
external_id: Mapped[str] = mapped_column(String(500))
|
|
source_schema_version: Mapped[str] = mapped_column(String(20))
|
|
url: Mapped[str] = mapped_column(Text)
|
|
content_type: Mapped[str | None] = mapped_column(String(200))
|
|
status_code: Mapped[int | None] = mapped_column(Integer)
|
|
body: Mapped[str] = mapped_column(Text)
|
|
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
retrieved_at: Mapped[datetime] = mapped_column(DateTime(timezone=True))
|
|
collected_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
|
|
|
|
|
|
class Document(Base):
|
|
__tablename__ = "documents"
|
|
__table_args__ = (
|
|
UniqueConstraint("corpus", "url", name="uq_document_corpus_url"),
|
|
UniqueConstraint("raw_document_id", name="uq_document_raw_document"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
raw_document_id: Mapped[int | None] = mapped_column(
|
|
ForeignKey("raw_documents.id", ondelete="SET NULL")
|
|
)
|
|
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
|
url: Mapped[str] = mapped_column(Text)
|
|
title: Mapped[str | None] = mapped_column(Text)
|
|
author: Mapped[str | None] = mapped_column(Text)
|
|
published_at: Mapped[str | None] = mapped_column(String(100))
|
|
text: Mapped[str] = mapped_column(Text)
|
|
content_hash: Mapped[str] = mapped_column(String(64), index=True)
|
|
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
is_duplicate: Mapped[bool] = mapped_column(default=False)
|
|
canonical_document_id: Mapped[int | None] = mapped_column(
|
|
ForeignKey("documents.id", ondelete="SET NULL")
|
|
)
|
|
|
|
|
|
class Chunk(Base):
|
|
__tablename__ = "chunks"
|
|
__table_args__ = (UniqueConstraint("document_id", "ordinal", name="uq_chunk_ordinal"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
document_id: Mapped[int] = mapped_column(
|
|
ForeignKey("documents.id", ondelete="CASCADE"), index=True
|
|
)
|
|
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
|
ordinal: Mapped[int] = mapped_column(Integer)
|
|
text: Mapped[str] = mapped_column(Text)
|
|
content_hash: Mapped[str] = mapped_column(String(64), index=True)
|
|
|
|
|
|
class KnowledgeModel(Base):
|
|
__tablename__ = "knowledge_models"
|
|
|
|
corpus: Mapped[str] = mapped_column(String(200), primary_key=True)
|
|
definition_json: Mapped[dict[str, Any]] = mapped_column(JSON)
|
|
updated_at: Mapped[datetime] = mapped_column(
|
|
DateTime(timezone=True), default=utc_now, onupdate=utc_now
|
|
)
|
|
|
|
|
|
class Extraction(Base):
|
|
__tablename__ = "extractions"
|
|
__table_args__ = (UniqueConstraint("chunk_id", name="uq_extraction_chunk"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
chunk_id: Mapped[int] = mapped_column(
|
|
ForeignKey("chunks.id", ondelete="CASCADE"), index=True
|
|
)
|
|
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
|
payload_json: Mapped[dict[str, Any]] = mapped_column(JSON)
|
|
model_name: Mapped[str] = mapped_column(String(200))
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
|
|
|
|
|
|
class ResolvedEntity(Base):
|
|
__tablename__ = "resolved_entities"
|
|
__table_args__ = (
|
|
UniqueConstraint("corpus", "canonical_key", name="uq_resolved_entity_key"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
|
canonical_key: Mapped[str] = mapped_column(String(500))
|
|
canonical_name: Mapped[str] = mapped_column(Text)
|
|
entity_type: Mapped[str] = mapped_column(String(100))
|
|
description: Mapped[str] = mapped_column(Text, default="")
|
|
aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list)
|
|
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
|
|
|
|
|
|
class ResolvedRelationship(Base):
|
|
__tablename__ = "resolved_relationships"
|
|
__table_args__ = (
|
|
UniqueConstraint(
|
|
"corpus",
|
|
"source_key",
|
|
"target_key",
|
|
"relationship_type",
|
|
name="uq_resolved_relationship",
|
|
),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
|
source_key: Mapped[str] = mapped_column(String(500))
|
|
target_key: Mapped[str] = mapped_column(String(500))
|
|
relationship_type: Mapped[str] = mapped_column(String(100))
|
|
description: Mapped[str] = mapped_column(Text, default="")
|
|
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
|
|
|
|
|
|
class Node(Base):
|
|
__tablename__ = "nodes"
|
|
__table_args__ = (
|
|
UniqueConstraint("corpus", "canonical_key", name="uq_node_canonical_key"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
|
canonical_key: Mapped[str] = mapped_column(String(500))
|
|
canonical_name: Mapped[str] = mapped_column(Text)
|
|
entity_type: Mapped[str] = mapped_column(String(100), index=True)
|
|
description: Mapped[str] = mapped_column(Text, default="")
|
|
aliases_json: Mapped[list[str]] = mapped_column(JSON, default=list)
|
|
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
|
|
|
|
|
|
class Edge(Base):
|
|
__tablename__ = "edges"
|
|
__table_args__ = (
|
|
UniqueConstraint(
|
|
"corpus",
|
|
"source_node_id",
|
|
"target_node_id",
|
|
"relationship_type",
|
|
name="uq_edge_identity",
|
|
),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
|
source_node_id: Mapped[int] = mapped_column(
|
|
ForeignKey("nodes.id", ondelete="CASCADE"), index=True
|
|
)
|
|
target_node_id: Mapped[int] = mapped_column(
|
|
ForeignKey("nodes.id", ondelete="CASCADE"), index=True
|
|
)
|
|
relationship_type: Mapped[str] = mapped_column(String(100), index=True)
|
|
description: Mapped[str] = mapped_column(Text, default="")
|
|
weight: Mapped[float] = mapped_column(Float, default=1.0)
|
|
source_chunk_ids_json: Mapped[list[int]] = mapped_column(JSON, default=list)
|
|
|
|
|
|
class Community(Base):
|
|
__tablename__ = "communities"
|
|
__table_args__ = (UniqueConstraint("corpus", "community_key", name="uq_community_key"),)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
|
community_key: Mapped[str] = mapped_column(String(100))
|
|
title: Mapped[str] = mapped_column(Text)
|
|
summary: Mapped[str] = mapped_column(Text)
|
|
node_ids_json: Mapped[list[int]] = mapped_column(JSON)
|
|
|
|
|
|
class SearchItem(Base):
|
|
__tablename__ = "retrieval_items"
|
|
__table_args__ = (
|
|
UniqueConstraint("corpus", "item_type", "item_key", name="uq_search_item"),
|
|
Index("ix_retrieval_items_corpus_type", "corpus", "item_type"),
|
|
Index(
|
|
"ix_retrieval_items_corpus_model",
|
|
"corpus",
|
|
"embedding_model",
|
|
),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
|
item_type: Mapped[str] = mapped_column(String(50))
|
|
item_key: Mapped[str] = mapped_column(String(100))
|
|
text: Mapped[str] = mapped_column(Text)
|
|
source_url: Mapped[str | None] = mapped_column(Text)
|
|
metadata_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
embedding: Mapped[list[float]] = mapped_column(
|
|
Vector().with_variant(JSON(), "sqlite")
|
|
)
|
|
embedding_model: Mapped[str] = mapped_column(String(200))
|
|
|
|
|
|
class PipelineRun(Base):
|
|
__tablename__ = "pipeline_runs"
|
|
__table_args__ = (
|
|
Index("ix_pipeline_runs_status_created", "status", "created_at"),
|
|
Index("ix_pipeline_runs_corpus_created", "corpus", "created_at"),
|
|
)
|
|
|
|
id: Mapped[str] = mapped_column(
|
|
String(36), primary_key=True, default=lambda: str(uuid4())
|
|
)
|
|
corpus: Mapped[str] = mapped_column(String(200), index=True)
|
|
status: Mapped[str] = mapped_column(String(30), index=True, default="queued")
|
|
from_stage: Mapped[int] = mapped_column(Integer)
|
|
through_stage: Mapped[int] = mapped_column(Integer)
|
|
payload_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
configuration_hash: Mapped[str] = mapped_column(String(64))
|
|
requested_by: Mapped[str] = mapped_column(String(500))
|
|
correlation_id: Mapped[str] = mapped_column(String(200), index=True)
|
|
feature_name: Mapped[str] = mapped_column(String(200), default="KnowledgeGraph")
|
|
permission_mask: Mapped[int] = mapped_column(Integer)
|
|
attempt: Mapped[int] = mapped_column(Integer, default=0)
|
|
output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
error_message: Mapped[str | None] = mapped_column(Text)
|
|
created_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
|
|
started_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
heartbeat_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True))
|
|
|
|
|
|
class StageRun(Base):
|
|
__tablename__ = "stage_runs"
|
|
__table_args__ = (
|
|
UniqueConstraint(
|
|
"pipeline_run_id",
|
|
"stage_number",
|
|
"attempt",
|
|
name="uq_stage_run_attempt",
|
|
),
|
|
Index("ix_stage_runs_pipeline_stage", "pipeline_run_id", "stage_number"),
|
|
)
|
|
|
|
id: Mapped[int] = mapped_column(primary_key=True)
|
|
pipeline_run_id: Mapped[str] = mapped_column(
|
|
ForeignKey("pipeline_runs.id", ondelete="CASCADE"), index=True
|
|
)
|
|
stage_number: Mapped[int] = mapped_column(Integer)
|
|
stage_name: Mapped[str] = mapped_column(String(200))
|
|
status: Mapped[str] = mapped_column(String(30), default="running")
|
|
attempt: Mapped[int] = mapped_column(Integer)
|
|
input_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
output_json: Mapped[dict[str, Any]] = mapped_column(JSON, default=dict)
|
|
error_message: Mapped[str | None] = mapped_column(Text)
|
|
started_at: Mapped[datetime] = mapped_column(DateTime(timezone=True), default=utc_now)
|
|
finished_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) |