84 lines
2.4 KiB
Python
84 lines
2.4 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from collections import Counter
|
|
from pathlib import Path
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json"
|
|
|
|
ALLOWED_CATEGORIES = {
|
|
"MEDICAL_ALLERGY",
|
|
"MEDICAL_DIAGNOSIS",
|
|
"MEDICAL_FAMILY_HISTORY",
|
|
"MEDICAL_MEDICATION",
|
|
"MEDICAL_MENTAL_HEALTH",
|
|
"MEDICAL_OTHER",
|
|
"MEDICAL_PHYSICAL_HEALTH",
|
|
"MEDICAL_PROCEDURE",
|
|
"MEDICAL_SUBSTANCE_USE",
|
|
"MEDICAL_SYMPTOM",
|
|
"MEDICAL_TEST",
|
|
"MEDICAL_TREATMENT",
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
source_path = PROJECT_ROOT / config["paths"]["source_export"]
|
|
|
|
ids: set[int] = set()
|
|
counts: Counter[tuple[str, bool]] = Counter()
|
|
|
|
with source_path.open("r", encoding="utf-8") as source:
|
|
for line_number, line in enumerate(source, start=1):
|
|
record = json.loads(line)
|
|
|
|
if set(record) != {"id", "category", "is_positive", "value"}:
|
|
raise RuntimeError(f"Invalid fields at line {line_number}")
|
|
|
|
record_id = record["id"]
|
|
|
|
if type(record_id) is not int or record_id <= 0:
|
|
raise RuntimeError(f"Invalid id at line {line_number}")
|
|
|
|
if record_id in ids:
|
|
raise RuntimeError(f"Duplicate id: {record_id}")
|
|
|
|
if record["category"] not in ALLOWED_CATEGORIES:
|
|
raise RuntimeError(
|
|
f"Invalid category at line {line_number}"
|
|
)
|
|
|
|
if type(record["is_positive"]) is not bool:
|
|
raise RuntimeError(
|
|
f"Invalid label at line {line_number}"
|
|
)
|
|
|
|
if not isinstance(record["value"], str) or not record["value"].strip():
|
|
raise RuntimeError(
|
|
f"Empty value at line {line_number}"
|
|
)
|
|
|
|
ids.add(record_id)
|
|
counts[(record["category"], record["is_positive"])] += 1
|
|
|
|
positive = sum(
|
|
count for (_, is_positive), count in counts.items()
|
|
if is_positive
|
|
)
|
|
negative = sum(
|
|
count for (_, is_positive), count in counts.items()
|
|
if not is_positive
|
|
)
|
|
|
|
print("Source validation passed.")
|
|
print(f"Total: {len(ids)}")
|
|
print(f"Positive: {positive}")
|
|
print(f"Negative: {negative}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|