from __future__ import annotations import json import random from collections import defaultdict from pathlib import Path PROJECT_ROOT = Path(__file__).resolve().parent.parent CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json" def read_jsonl(path: Path) -> list[dict]: with path.open("r", encoding="utf-8") as file: return [json.loads(line) for line in file if line.strip()] def write_jsonl(path: Path, records: list[dict]) -> None: with path.open("w", encoding="utf-8", newline="\n") as file: for record in records: file.write(json.dumps(record, separators=(",", ":")) + "\n") def main() -> None: config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) paths = config["paths"] split_config = config["splits"] source_path = PROJECT_ROOT / paths["source_export"] records = read_jsonl(source_path) grouped: dict[tuple[str, bool], list[dict]] = defaultdict(list) for record in records: grouped[(record["category"], record["is_positive"])].append(record) train: list[dict] = [] validation: list[dict] = [] development: list[dict] = [] seed = int(split_config["seed"]) for key in sorted(grouped): group = grouped[key] randomizer = random.Random(f"{seed}:{key[0]}:{key[1]}") randomizer.shuffle(group) count = len(group) validation_count = int(count * split_config["validation"] + 0.5) development_count = int(count * split_config["development"] + 0.5) train_count = count - validation_count - development_count train.extend(group[:train_count]) validation.extend( group[train_count:train_count + validation_count] ) development.extend( group[train_count + validation_count:] ) random.Random(f"{seed}:train").shuffle(train) random.Random(f"{seed}:validation").shuffle(validation) random.Random(f"{seed}:development").shuffle(development) outputs = { "train": train, "validation": validation, "development": development, } all_keys: set[str] = set() for name, split_records in outputs.items(): split_keys = {record["id"] for record in split_records} if len(split_keys) != len(split_records): raise RuntimeError(f"Duplicate id inside {name}") overlap = all_keys.intersection(split_keys) if overlap: raise RuntimeError(f"Split leakage detected in {name}") all_keys.update(split_keys) write_jsonl(PROJECT_ROOT / paths[name], split_records) if len(all_keys) != len(records): raise RuntimeError("Not every source record was assigned exactly once") print("Split creation passed.") print(f"Source: {len(records)}") print(f"Train: {len(train)}") print(f"Validation: {len(validation)}") print(f"Development: {len(development)}") print(f"Seed: {seed}") if __name__ == "__main__": main()