Models-Phi-8B/scripts/create_splits.py

100 lines
2.9 KiB
Python

from __future__ import annotations
import json
import random
from collections import defaultdict
from pathlib import Path
PROJECT_ROOT = Path(__file__).resolve().parent.parent
CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json"
def read_jsonl(path: Path) -> list[dict]:
with path.open("r", encoding="utf-8") as file:
return [json.loads(line) for line in file if line.strip()]
def write_jsonl(path: Path, records: list[dict]) -> None:
with path.open("w", encoding="utf-8", newline="\n") as file:
for record in records:
file.write(json.dumps(record, separators=(",", ":")) + "\n")
def main() -> None:
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
paths = config["paths"]
split_config = config["splits"]
source_path = PROJECT_ROOT / paths["source_export"]
records = read_jsonl(source_path)
grouped: dict[tuple[str, bool], list[dict]] = defaultdict(list)
for record in records:
grouped[(record["category"], record["is_positive"])].append(record)
train: list[dict] = []
validation: list[dict] = []
development: list[dict] = []
seed = int(split_config["seed"])
for key in sorted(grouped):
group = grouped[key]
randomizer = random.Random(f"{seed}:{key[0]}:{key[1]}")
randomizer.shuffle(group)
count = len(group)
validation_count = int(count * split_config["validation"] + 0.5)
development_count = int(count * split_config["development"] + 0.5)
train_count = count - validation_count - development_count
train.extend(group[:train_count])
validation.extend(
group[train_count:train_count + validation_count]
)
development.extend(
group[train_count + validation_count:]
)
random.Random(f"{seed}:train").shuffle(train)
random.Random(f"{seed}:validation").shuffle(validation)
random.Random(f"{seed}:development").shuffle(development)
outputs = {
"train": train,
"validation": validation,
"development": development,
}
all_keys: set[str] = set()
for name, split_records in outputs.items():
split_keys = {record["id"] for record in split_records}
if len(split_keys) != len(split_records):
raise RuntimeError(f"Duplicate id inside {name}")
overlap = all_keys.intersection(split_keys)
if overlap:
raise RuntimeError(f"Split leakage detected in {name}")
all_keys.update(split_keys)
write_jsonl(PROJECT_ROOT / paths[name], split_records)
if len(all_keys) != len(records):
raise RuntimeError("Not every source record was assigned exactly once")
print("Split creation passed.")
print(f"Source: {len(records)}")
print(f"Train: {len(train)}")
print(f"Validation: {len(validation)}")
print(f"Development: {len(development)}")
print(f"Seed: {seed}")
if __name__ == "__main__":
main()