Models-Phi-8B/scripts/build_sft.py

111 lines
2.9 KiB
Python

from __future__ import annotations
import json
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parent.parent
CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json"
SYSTEM_PROMPT_PATH = PROJECT_ROOT / "config" / "system_prompt.txt"
def read_jsonl(path: Path) -> list[dict]:
with path.open("r", encoding="utf-8") as file:
return [json.loads(line) for line in file if line.strip()]
def write_jsonl(path: Path, records: list[dict]) -> None:
with path.open("w", encoding="utf-8", newline="\n") as file:
for record in records:
file.write(
json.dumps(record, ensure_ascii=False, separators=(",", ":"))
+ "\n"
)
def build_target(record: dict) -> dict:
value = record["value"]
category = record["category"]
if record["is_positive"]:
return {
"is_medical_phi": True,
"score": 1.0,
"reason": (
"The input contains explicit personal medical information."
),
"results": [
{
"entity_type": category,
"text": value,
"score": 1.0,
"evidence": value,
}
],
}
return {
"is_medical_phi": False,
"score": 0.0,
"reason": (
"The input does not contain personal medical information "
"under the configured policy."
),
"results": [],
}
def build_record(record: dict, system_prompt: str) -> dict:
target = build_target(record)
return {
"id": record["id"],
"messages": [
{
"role": "system",
"content": system_prompt,
},
{
"role": "user",
"content": record["value"],
},
{
"role": "assistant",
"content": json.dumps(
target,
ensure_ascii=False,
separators=(",", ":"),
),
},
],
}
def main() -> None:
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
system_prompt = SYSTEM_PROMPT_PATH.read_text(
encoding="utf-8"
).strip()
for split_name in ("train", "validation", "development"):
path = PROJECT_ROOT / config["paths"][split_name]
source_records = read_jsonl(path)
sft_records = [
build_record(record, system_prompt)
for record in source_records
]
for record in sft_records:
assistant_output = record["messages"][2]["content"]
json.loads(assistant_output)
write_jsonl(path, sft_records)
print(f"{split_name}: {len(sft_records)}")
print("SFT dataset creation passed.")
if __name__ == "__main__":
main()