Models-Phi-8B/scripts/evaluate.py

494 lines
13 KiB
Python

from __future__ import annotations
import json
import shutil
from pathlib import Path
from typing import Any
import torch
from peft import PeftModel
from tqdm import tqdm
from json_stopping import JsonObjectStoppingCriteria
from transformers import (
StoppingCriteriaList,
AutoModelForCausalLM,
AutoTokenizer,
BitsAndBytesConfig,
)
PROJECT_ROOT = Path(__file__).resolve().parent.parent
CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json"
CHAT_TEMPLATE_PATH = PROJECT_ROOT / "config" / "chat_template.jinja"
ALLOWED_ENTITY_TYPES = {
"MEDICAL_ALLERGY",
"MEDICAL_DIAGNOSIS",
"MEDICAL_FAMILY_HISTORY",
"MEDICAL_MEDICATION",
"MEDICAL_MENTAL_HEALTH",
"MEDICAL_OTHER",
"MEDICAL_PHYSICAL_HEALTH",
"MEDICAL_PROCEDURE",
"MEDICAL_SUBSTANCE_USE",
"MEDICAL_SYMPTOM",
"MEDICAL_TEST",
"MEDICAL_TREATMENT",
}
def read_jsonl(path: Path) -> list[dict[str, Any]]:
records: list[dict[str, Any]] = []
with path.open("r", encoding="utf-8") as file:
for line in file:
if line.strip():
records.append(json.loads(line))
return records
def is_number(value: Any) -> bool:
return (
isinstance(value, (int, float))
and not isinstance(value, bool)
)
def valid_score(value: Any) -> bool:
return is_number(value) and 0.0 <= float(value) <= 1.0
def validate_schema(value: Any) -> bool:
if not isinstance(value, dict):
return False
if set(value) != {
"is_medical_phi",
"score",
"reason",
"results",
}:
return False
if not isinstance(value["is_medical_phi"], bool):
return False
if not valid_score(value["score"]):
return False
if not isinstance(value["reason"], str):
return False
if not isinstance(value["results"], list):
return False
for result in value["results"]:
if not isinstance(result, dict):
return False
if set(result) != {
"entity_type",
"text",
"score",
"evidence",
}:
return False
if result["entity_type"] not in ALLOWED_ENTITY_TYPES:
return False
if not isinstance(result["text"], str):
return False
if not valid_score(result["score"]):
return False
if not isinstance(result["evidence"], str):
return False
return True
def safe_divide(numerator: int, denominator: int) -> float:
if denominator == 0:
return 0.0
return numerator / denominator
def main() -> None:
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
model_path = config["model"]["base_model"]
max_length = int(config["model"]["max_sequence_length"])
adapter_path = (
PROJECT_ROOT
/ config["training"]["full"]["output_dir"]
)
development_path = (
PROJECT_ROOT
/ config["paths"]["development"]
)
output_dir = PROJECT_ROOT / "outputs" / "development-evaluation"
temporary_dir = PROJECT_ROOT / "outputs" / "development-evaluation.tmp"
if temporary_dir.exists():
shutil.rmtree(temporary_dir)
temporary_dir.mkdir(parents=True)
predictions_path = temporary_dir / "predictions.jsonl"
summary_path = temporary_dir / "summary.json"
records = read_jsonl(development_path)
print(f"Development records: {len(records)}")
print("Loading tokenizer...")
tokenizer = AutoTokenizer.from_pretrained(
model_path,
use_fast=True,
)
tokenizer.chat_template = CHAT_TEMPLATE_PATH.read_text(
encoding="utf-8"
)
if tokenizer.pad_token_id is None:
tokenizer.pad_token = tokenizer.eos_token
tokenizer.padding_side = "left"
quantization_config = BitsAndBytesConfig(
load_in_4bit=True,
bnb_4bit_quant_type="nf4",
bnb_4bit_use_double_quant=True,
bnb_4bit_compute_dtype=torch.bfloat16,
)
print("Loading base model in 4-bit NF4...")
base_model = AutoModelForCausalLM.from_pretrained(
model_path,
quantization_config=quantization_config,
dtype=torch.bfloat16,
device_map={"": 0},
low_cpu_mem_usage=True,
)
print(f"Loading adapter: {adapter_path}")
model = PeftModel.from_pretrained(
base_model,
adapter_path,
)
model.eval()
model.config.use_cache = True
eos_token_ids = [tokenizer.eos_token_id]
eot_token_id = tokenizer.convert_tokens_to_ids("<|eot_id|>")
if (
isinstance(eot_token_id, int)
and eot_token_id >= 0
and eot_token_id not in eos_token_ids
):
eos_token_ids.append(eot_token_id)
total = len(records)
valid_json_count = 0
valid_schema_count = 0
classification_correct = 0
result_count_correct = 0
true_positive = 0
true_negative = 0
false_positive = 0
false_negative = 0
positive_total = 0
entity_type_correct = 0
entity_text_correct = 0
evidence_correct = 0
batch_size = 8
max_new_tokens = 256
print("Running deterministic development evaluation...")
with predictions_path.open(
"w",
encoding="utf-8",
newline="\n",
) as prediction_file:
for start in tqdm(
range(0, total, batch_size),
desc="Evaluating",
):
batch = records[start:start + batch_size]
prompts = [
tokenizer.apply_chat_template(
record["messages"][:2],
tokenize=False,
add_generation_prompt=True,
)
for record in batch
]
encoded = tokenizer(
prompts,
return_tensors="pt",
padding=True,
truncation=True,
max_length=max_length,
)
encoded = {
name: tensor.to(model.device)
for name, tensor in encoded.items()
}
prompt_length = encoded["input_ids"].shape[1]
stopping_criteria = StoppingCriteriaList(
[
JsonObjectStoppingCriteria(
tokenizer,
prompt_length,
)
]
)
with torch.inference_mode():
generated = model.generate(
**encoded,
do_sample=False,
max_new_tokens=max_new_tokens,
eos_token_id=eos_token_ids,
pad_token_id=tokenizer.pad_token_id,
stopping_criteria=stopping_criteria,
use_cache=True,
)
generated_tokens = generated[:, prompt_length:]
raw_outputs = tokenizer.batch_decode(
generated_tokens,
skip_special_tokens=True,
clean_up_tokenization_spaces=False,
)
for record, raw_output in zip(batch, raw_outputs):
raw_output = raw_output.strip()
expected = json.loads(
record["messages"][2]["content"]
)
parsed: Any = None
json_valid = False
schema_valid = False
try:
parsed, _ = json.JSONDecoder().raw_decode(
raw_output.lstrip()
)
json_valid = True
valid_json_count += 1
except json.JSONDecodeError:
pass
if json_valid and validate_schema(parsed):
schema_valid = True
valid_schema_count += 1
gold_phi = expected["is_medical_phi"]
if gold_phi:
positive_total += 1
if schema_valid:
predicted_phi = parsed["is_medical_phi"]
if predicted_phi == gold_phi:
classification_correct += 1
if gold_phi and predicted_phi:
true_positive += 1
elif not gold_phi and not predicted_phi:
true_negative += 1
elif not gold_phi and predicted_phi:
false_positive += 1
else:
false_negative += 1
expected_results = expected["results"]
predicted_results = parsed["results"]
if len(predicted_results) == len(expected_results):
result_count_correct += 1
if gold_phi and expected_results:
expected_result = expected_results[0]
if predicted_results:
predicted_result = predicted_results[0]
if (
predicted_result["entity_type"]
== expected_result["entity_type"]
):
entity_type_correct += 1
if (
predicted_result["text"]
== expected_result["text"]
):
entity_text_correct += 1
if (
predicted_result["evidence"]
== expected_result["evidence"]
):
evidence_correct += 1
prediction_record = {
"id": record["id"],
"input": record["messages"][1]["content"],
"expected": expected,
"raw_output": raw_output,
"parsed_output": parsed,
"json_valid": json_valid,
"schema_valid": schema_valid,
}
prediction_file.write(
json.dumps(
prediction_record,
ensure_ascii=False,
separators=(",", ":"),
)
+ "\n"
)
precision = safe_divide(
true_positive,
true_positive + false_positive,
)
recall = safe_divide(
true_positive,
true_positive + false_negative,
)
f1 = safe_divide(
2 * precision * recall,
precision + recall,
)
summary = {
"total_records": total,
"valid_json": valid_json_count,
"valid_json_rate": safe_divide(
valid_json_count,
total,
),
"valid_schema": valid_schema_count,
"valid_schema_rate": safe_divide(
valid_schema_count,
total,
),
"classification_correct": classification_correct,
"classification_accuracy": safe_divide(
classification_correct,
total,
),
"true_positive": true_positive,
"true_negative": true_negative,
"false_positive": false_positive,
"false_negative": false_negative,
"precision": precision,
"recall": recall,
"f1": f1,
"result_count_correct": result_count_correct,
"result_count_accuracy": safe_divide(
result_count_correct,
total,
),
"positive_records": positive_total,
"entity_type_correct": entity_type_correct,
"entity_type_accuracy": safe_divide(
entity_type_correct,
positive_total,
),
"entity_text_correct": entity_text_correct,
"entity_text_accuracy": safe_divide(
entity_text_correct,
positive_total,
),
"evidence_correct": evidence_correct,
"evidence_accuracy": safe_divide(
evidence_correct,
positive_total,
),
}
summary_path.write_text(
json.dumps(summary, indent=2) + "\n",
encoding="utf-8",
)
if output_dir.exists():
shutil.rmtree(output_dir)
temporary_dir.rename(output_dir)
print()
print("Development evaluation completed.")
print(f"Total records: {total}")
print(
"Valid JSON: "
f"{valid_json_count}/{total} "
f"({summary['valid_json_rate']:.4%})"
)
print(
"Valid schema: "
f"{valid_schema_count}/{total} "
f"({summary['valid_schema_rate']:.4%})"
)
print(
"Classification accuracy:"
f" {summary['classification_accuracy']:.4%}"
)
print(f"Precision: {precision:.4%}")
print(f"Recall: {recall:.4%}")
print(f"F1: {f1:.4%}")
print(
"Result count accuracy: "
f"{summary['result_count_accuracy']:.4%}"
)
print(
"Entity type accuracy: "
f"{summary['entity_type_accuracy']:.4%}"
)
print(
"Entity text accuracy: "
f"{summary['entity_text_accuracy']:.4%}"
)
print(
"Evidence accuracy: "
f"{summary['evidence_accuracy']:.4%}"
)
print(f"Predictions: {output_dir / 'predictions.jsonl'}")
print(f"Summary: {output_dir / 'summary.json'}")
if __name__ == "__main__":
main()