100 lines
2.9 KiB
Python
100 lines
2.9 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import random
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json"
|
|
|
|
|
|
def read_jsonl(path: Path) -> list[dict]:
|
|
with path.open("r", encoding="utf-8") as file:
|
|
return [json.loads(line) for line in file if line.strip()]
|
|
|
|
|
|
def write_jsonl(path: Path, records: list[dict]) -> None:
|
|
with path.open("w", encoding="utf-8", newline="\n") as file:
|
|
for record in records:
|
|
file.write(json.dumps(record, separators=(",", ":")) + "\n")
|
|
|
|
|
|
def main() -> None:
|
|
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
paths = config["paths"]
|
|
split_config = config["splits"]
|
|
|
|
source_path = PROJECT_ROOT / paths["source_export"]
|
|
records = read_jsonl(source_path)
|
|
|
|
grouped: dict[tuple[str, bool], list[dict]] = defaultdict(list)
|
|
|
|
for record in records:
|
|
grouped[(record["category"], record["is_positive"])].append(record)
|
|
|
|
train: list[dict] = []
|
|
validation: list[dict] = []
|
|
development: list[dict] = []
|
|
|
|
seed = int(split_config["seed"])
|
|
|
|
for key in sorted(grouped):
|
|
group = grouped[key]
|
|
|
|
randomizer = random.Random(f"{seed}:{key[0]}:{key[1]}")
|
|
randomizer.shuffle(group)
|
|
|
|
count = len(group)
|
|
validation_count = int(count * split_config["validation"] + 0.5)
|
|
development_count = int(count * split_config["development"] + 0.5)
|
|
train_count = count - validation_count - development_count
|
|
|
|
train.extend(group[:train_count])
|
|
validation.extend(
|
|
group[train_count:train_count + validation_count]
|
|
)
|
|
development.extend(
|
|
group[train_count + validation_count:]
|
|
)
|
|
|
|
random.Random(f"{seed}:train").shuffle(train)
|
|
random.Random(f"{seed}:validation").shuffle(validation)
|
|
random.Random(f"{seed}:development").shuffle(development)
|
|
|
|
outputs = {
|
|
"train": train,
|
|
"validation": validation,
|
|
"development": development,
|
|
}
|
|
|
|
all_keys: set[str] = set()
|
|
|
|
for name, split_records in outputs.items():
|
|
split_keys = {record["id"] for record in split_records}
|
|
|
|
if len(split_keys) != len(split_records):
|
|
raise RuntimeError(f"Duplicate id inside {name}")
|
|
|
|
overlap = all_keys.intersection(split_keys)
|
|
if overlap:
|
|
raise RuntimeError(f"Split leakage detected in {name}")
|
|
|
|
all_keys.update(split_keys)
|
|
write_jsonl(PROJECT_ROOT / paths[name], split_records)
|
|
|
|
if len(all_keys) != len(records):
|
|
raise RuntimeError("Not every source record was assigned exactly once")
|
|
|
|
print("Split creation passed.")
|
|
print(f"Source: {len(records)}")
|
|
print(f"Train: {len(train)}")
|
|
print(f"Validation: {len(validation)}")
|
|
print(f"Development: {len(development)}")
|
|
print(f"Seed: {seed}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|