from __future__ import annotations import json from collections import Counter from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parent.parent CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json" ALLOWED_CATEGORIES = { "MEDICAL_ALLERGY", "MEDICAL_DIAGNOSIS", "MEDICAL_FAMILY_HISTORY", "MEDICAL_MEDICATION", "MEDICAL_MENTAL_HEALTH", "MEDICAL_OTHER", "MEDICAL_PHYSICAL_HEALTH", "MEDICAL_PROCEDURE", "MEDICAL_SUBSTANCE_USE", "MEDICAL_SYMPTOM", "MEDICAL_TEST", "MEDICAL_TREATMENT", } def main() -> None: config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) source_path = PROJECT_ROOT / config["paths"]["source_export"] ids: set[int] = set() counts: Counter[tuple[str, bool]] = Counter() with source_path.open("r", encoding="utf-8") as source: for line_number, line in enumerate(source, start=1): record = json.loads(line) if set(record) != {"id", "category", "is_positive", "value"}: raise RuntimeError(f"Invalid fields at line {line_number}") record_id = record["id"] if type(record_id) is not int or record_id <= 0: raise RuntimeError(f"Invalid id at line {line_number}") if record_id in ids: raise RuntimeError(f"Duplicate id: {record_id}") if record["category"] not in ALLOWED_CATEGORIES: raise RuntimeError( f"Invalid category at line {line_number}" ) if type(record["is_positive"]) is not bool: raise RuntimeError( f"Invalid label at line {line_number}" ) if not isinstance(record["value"], str) or not record["value"].strip(): raise RuntimeError( f"Empty value at line {line_number}" ) ids.add(record_id) counts[(record["category"], record["is_positive"])] += 1 positive = sum( count for (_, is_positive), count in counts.items() if is_positive ) negative = sum( count for (_, is_positive), count in counts.items() if not is_positive ) print("Source validation passed.") print(f"Total: {len(ids)}") print(f"Positive: {positive}") print(f"Negative: {negative}") if __name__ == "__main__": main()