from __future__ import annotations import json from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parent.parent CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json" SYSTEM_PROMPT_PATH = PROJECT_ROOT / "config" / "system_prompt.txt" def read_jsonl(path: Path) -> list[dict]: with path.open("r", encoding="utf-8") as file: return [json.loads(line) for line in file if line.strip()] def write_jsonl(path: Path, records: list[dict]) -> None: with path.open("w", encoding="utf-8", newline="\n") as file: for record in records: file.write( json.dumps(record, ensure_ascii=False, separators=(",", ":")) + "\n" ) def build_target(record: dict) -> dict: value = record["value"] category = record["category"] if record["is_positive"]: return { "is_medical_phi": True, "score": 1.0, "reason": ( "The input contains explicit personal medical information." ), "results": [ { "entity_type": category, "text": value, "score": 1.0, "evidence": value, } ], } return { "is_medical_phi": False, "score": 0.0, "reason": ( "The input does not contain personal medical information " "under the configured policy." ), "results": [], } def build_record(record: dict, system_prompt: str) -> dict: target = build_target(record) return { "id": record["id"], "messages": [ { "role": "system", "content": system_prompt, }, { "role": "user", "content": record["value"], }, { "role": "assistant", "content": json.dumps( target, ensure_ascii=False, separators=(",", ":"), ), }, ], } def main() -> None: config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) system_prompt = SYSTEM_PROMPT_PATH.read_text( encoding="utf-8" ).strip() for split_name in ("train", "validation", "development"): path = PROJECT_ROOT / config["paths"][split_name] source_records = read_jsonl(path) sft_records = [ build_record(record, system_prompt) for record in source_records ] for record in sft_records: assistant_output = record["messages"][2]["content"] json.loads(assistant_output) write_jsonl(path, sft_records) print(f"{split_name}: {len(sft_records)}") print("SFT dataset creation passed.") if __name__ == "__main__": main()