Models-Phi-8B/scripts/validate_source.py

84 lines
2.4 KiB
Python

from __future__ import annotations
import json
from collections import Counter
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parent.parent
CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json"
ALLOWED_CATEGORIES = {
"MEDICAL_ALLERGY",
"MEDICAL_DIAGNOSIS",
"MEDICAL_FAMILY_HISTORY",
"MEDICAL_MEDICATION",
"MEDICAL_MENTAL_HEALTH",
"MEDICAL_OTHER",
"MEDICAL_PHYSICAL_HEALTH",
"MEDICAL_PROCEDURE",
"MEDICAL_SUBSTANCE_USE",
"MEDICAL_SYMPTOM",
"MEDICAL_TEST",
"MEDICAL_TREATMENT",
}
def main() -> None:
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
source_path = PROJECT_ROOT / config["paths"]["source_export"]
ids: set[int] = set()
counts: Counter[tuple[str, bool]] = Counter()
with source_path.open("r", encoding="utf-8") as source:
for line_number, line in enumerate(source, start=1):
record = json.loads(line)
if set(record) != {"id", "category", "is_positive", "value"}:
raise RuntimeError(f"Invalid fields at line {line_number}")
record_id = record["id"]
if type(record_id) is not int or record_id <= 0:
raise RuntimeError(f"Invalid id at line {line_number}")
if record_id in ids:
raise RuntimeError(f"Duplicate id: {record_id}")
if record["category"] not in ALLOWED_CATEGORIES:
raise RuntimeError(
f"Invalid category at line {line_number}"
)
if type(record["is_positive"]) is not bool:
raise RuntimeError(
f"Invalid label at line {line_number}"
)
if not isinstance(record["value"], str) or not record["value"].strip():
raise RuntimeError(
f"Empty value at line {line_number}"
)
ids.add(record_id)
counts[(record["category"], record["is_positive"])] += 1
positive = sum(
count for (_, is_positive), count in counts.items()
if is_positive
)
negative = sum(
count for (_, is_positive), count in counts.items()
if not is_positive
)
print("Source validation passed.")
print(f"Total: {len(ids)}")
print(f"Positive: {positive}")
print(f"Negative: {negative}")
if __name__ == "__main__":
main()