111 lines
2.9 KiB
Python
111 lines
2.9 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json"
|
|
SYSTEM_PROMPT_PATH = PROJECT_ROOT / "config" / "system_prompt.txt"
|
|
|
|
|
|
def read_jsonl(path: Path) -> list[dict]:
|
|
with path.open("r", encoding="utf-8") as file:
|
|
return [json.loads(line) for line in file if line.strip()]
|
|
|
|
|
|
def write_jsonl(path: Path, records: list[dict]) -> None:
|
|
with path.open("w", encoding="utf-8", newline="\n") as file:
|
|
for record in records:
|
|
file.write(
|
|
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
|
|
+ "\n"
|
|
)
|
|
|
|
|
|
def build_target(record: dict) -> dict:
|
|
value = record["value"]
|
|
category = record["category"]
|
|
|
|
if record["is_positive"]:
|
|
return {
|
|
"is_medical_phi": True,
|
|
"score": 1.0,
|
|
"reason": (
|
|
"The input contains explicit personal medical information."
|
|
),
|
|
"results": [
|
|
{
|
|
"entity_type": category,
|
|
"text": value,
|
|
"score": 1.0,
|
|
"evidence": value,
|
|
}
|
|
],
|
|
}
|
|
|
|
return {
|
|
"is_medical_phi": False,
|
|
"score": 0.0,
|
|
"reason": (
|
|
"The input does not contain personal medical information "
|
|
"under the configured policy."
|
|
),
|
|
"results": [],
|
|
}
|
|
|
|
|
|
def build_record(record: dict, system_prompt: str) -> dict:
|
|
target = build_target(record)
|
|
|
|
return {
|
|
"id": record["id"],
|
|
"messages": [
|
|
{
|
|
"role": "system",
|
|
"content": system_prompt,
|
|
},
|
|
{
|
|
"role": "user",
|
|
"content": record["value"],
|
|
},
|
|
{
|
|
"role": "assistant",
|
|
"content": json.dumps(
|
|
target,
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
),
|
|
},
|
|
],
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
system_prompt = SYSTEM_PROMPT_PATH.read_text(
|
|
encoding="utf-8"
|
|
).strip()
|
|
|
|
for split_name in ("train", "validation", "development"):
|
|
path = PROJECT_ROOT / config["paths"][split_name]
|
|
source_records = read_jsonl(path)
|
|
|
|
sft_records = [
|
|
build_record(record, system_prompt)
|
|
for record in source_records
|
|
]
|
|
|
|
for record in sft_records:
|
|
assistant_output = record["messages"][2]["content"]
|
|
json.loads(assistant_output)
|
|
|
|
write_jsonl(path, sft_records)
|
|
print(f"{split_name}: {len(sft_records)}")
|
|
|
|
print("SFT dataset creation passed.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|