from __future__ import annotations import json import shutil from pathlib import Path from typing import Any import torch from peft import PeftModel from tqdm import tqdm from json_stopping import JsonObjectStoppingCriteria from transformers import ( StoppingCriteriaList, AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig, ) PROJECT_ROOT = Path(__file__).resolve().parent.parent CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json" CHAT_TEMPLATE_PATH = PROJECT_ROOT / "config" / "chat_template.jinja" ALLOWED_ENTITY_TYPES = { "MEDICAL_ALLERGY", "MEDICAL_DIAGNOSIS", "MEDICAL_FAMILY_HISTORY", "MEDICAL_MEDICATION", "MEDICAL_MENTAL_HEALTH", "MEDICAL_OTHER", "MEDICAL_PHYSICAL_HEALTH", "MEDICAL_PROCEDURE", "MEDICAL_SUBSTANCE_USE", "MEDICAL_SYMPTOM", "MEDICAL_TEST", "MEDICAL_TREATMENT", } def read_jsonl(path: Path) -> list[dict[str, Any]]: records: list[dict[str, Any]] = [] with path.open("r", encoding="utf-8") as file: for line in file: if line.strip(): records.append(json.loads(line)) return records def is_number(value: Any) -> bool: return ( isinstance(value, (int, float)) and not isinstance(value, bool) ) def valid_score(value: Any) -> bool: return is_number(value) and 0.0 <= float(value) <= 1.0 def validate_schema(value: Any) -> bool: if not isinstance(value, dict): return False if set(value) != { "is_medical_phi", "score", "reason", "results", }: return False if not isinstance(value["is_medical_phi"], bool): return False if not valid_score(value["score"]): return False if not isinstance(value["reason"], str): return False if not isinstance(value["results"], list): return False for result in value["results"]: if not isinstance(result, dict): return False if set(result) != { "entity_type", "text", "score", "evidence", }: return False if result["entity_type"] not in ALLOWED_ENTITY_TYPES: return False if not isinstance(result["text"], str): return False if not valid_score(result["score"]): return False if not isinstance(result["evidence"], str): return False return True def safe_divide(numerator: int, denominator: int) -> float: if denominator == 0: return 0.0 return numerator / denominator def main() -> None: config = json.loads(CONFIG_PATH.read_text(encoding="utf-8")) model_path = config["model"]["base_model"] max_length = int(config["model"]["max_sequence_length"]) adapter_path = ( PROJECT_ROOT / config["training"]["full"]["output_dir"] ) development_path = ( PROJECT_ROOT / config["paths"]["development"] ) output_dir = PROJECT_ROOT / "outputs" / "development-evaluation" temporary_dir = PROJECT_ROOT / "outputs" / "development-evaluation.tmp" if temporary_dir.exists(): shutil.rmtree(temporary_dir) temporary_dir.mkdir(parents=True) predictions_path = temporary_dir / "predictions.jsonl" summary_path = temporary_dir / "summary.json" records = read_jsonl(development_path) print(f"Development records: {len(records)}") print("Loading tokenizer...") tokenizer = AutoTokenizer.from_pretrained( model_path, use_fast=True, ) tokenizer.chat_template = CHAT_TEMPLATE_PATH.read_text( encoding="utf-8" ) if tokenizer.pad_token_id is None: tokenizer.pad_token = tokenizer.eos_token tokenizer.padding_side = "left" quantization_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_quant_type="nf4", bnb_4bit_use_double_quant=True, bnb_4bit_compute_dtype=torch.bfloat16, ) print("Loading base model in 4-bit NF4...") base_model = AutoModelForCausalLM.from_pretrained( model_path, quantization_config=quantization_config, dtype=torch.bfloat16, device_map={"": 0}, low_cpu_mem_usage=True, ) print(f"Loading adapter: {adapter_path}") model = PeftModel.from_pretrained( base_model, adapter_path, ) model.eval() model.config.use_cache = True eos_token_ids = [tokenizer.eos_token_id] eot_token_id = tokenizer.convert_tokens_to_ids("<|eot_id|>") if ( isinstance(eot_token_id, int) and eot_token_id >= 0 and eot_token_id not in eos_token_ids ): eos_token_ids.append(eot_token_id) total = len(records) valid_json_count = 0 valid_schema_count = 0 classification_correct = 0 result_count_correct = 0 true_positive = 0 true_negative = 0 false_positive = 0 false_negative = 0 positive_total = 0 entity_type_correct = 0 entity_text_correct = 0 evidence_correct = 0 batch_size = 8 max_new_tokens = 256 print("Running deterministic development evaluation...") with predictions_path.open( "w", encoding="utf-8", newline="\n", ) as prediction_file: for start in tqdm( range(0, total, batch_size), desc="Evaluating", ): batch = records[start:start + batch_size] prompts = [ tokenizer.apply_chat_template( record["messages"][:2], tokenize=False, add_generation_prompt=True, ) for record in batch ] encoded = tokenizer( prompts, return_tensors="pt", padding=True, truncation=True, max_length=max_length, ) encoded = { name: tensor.to(model.device) for name, tensor in encoded.items() } prompt_length = encoded["input_ids"].shape[1] stopping_criteria = StoppingCriteriaList( [ JsonObjectStoppingCriteria( tokenizer, prompt_length, ) ] ) with torch.inference_mode(): generated = model.generate( **encoded, do_sample=False, max_new_tokens=max_new_tokens, eos_token_id=eos_token_ids, pad_token_id=tokenizer.pad_token_id, stopping_criteria=stopping_criteria, use_cache=True, ) generated_tokens = generated[:, prompt_length:] raw_outputs = tokenizer.batch_decode( generated_tokens, skip_special_tokens=True, clean_up_tokenization_spaces=False, ) for record, raw_output in zip(batch, raw_outputs): raw_output = raw_output.strip() expected = json.loads( record["messages"][2]["content"] ) parsed: Any = None json_valid = False schema_valid = False try: parsed, _ = json.JSONDecoder().raw_decode( raw_output.lstrip() ) json_valid = True valid_json_count += 1 except json.JSONDecodeError: pass if json_valid and validate_schema(parsed): schema_valid = True valid_schema_count += 1 gold_phi = expected["is_medical_phi"] if gold_phi: positive_total += 1 if schema_valid: predicted_phi = parsed["is_medical_phi"] if predicted_phi == gold_phi: classification_correct += 1 if gold_phi and predicted_phi: true_positive += 1 elif not gold_phi and not predicted_phi: true_negative += 1 elif not gold_phi and predicted_phi: false_positive += 1 else: false_negative += 1 expected_results = expected["results"] predicted_results = parsed["results"] if len(predicted_results) == len(expected_results): result_count_correct += 1 if gold_phi and expected_results: expected_result = expected_results[0] if predicted_results: predicted_result = predicted_results[0] if ( predicted_result["entity_type"] == expected_result["entity_type"] ): entity_type_correct += 1 if ( predicted_result["text"] == expected_result["text"] ): entity_text_correct += 1 if ( predicted_result["evidence"] == expected_result["evidence"] ): evidence_correct += 1 prediction_record = { "id": record["id"], "input": record["messages"][1]["content"], "expected": expected, "raw_output": raw_output, "parsed_output": parsed, "json_valid": json_valid, "schema_valid": schema_valid, } prediction_file.write( json.dumps( prediction_record, ensure_ascii=False, separators=(",", ":"), ) + "\n" ) precision = safe_divide( true_positive, true_positive + false_positive, ) recall = safe_divide( true_positive, true_positive + false_negative, ) f1 = safe_divide( 2 * precision * recall, precision + recall, ) summary = { "total_records": total, "valid_json": valid_json_count, "valid_json_rate": safe_divide( valid_json_count, total, ), "valid_schema": valid_schema_count, "valid_schema_rate": safe_divide( valid_schema_count, total, ), "classification_correct": classification_correct, "classification_accuracy": safe_divide( classification_correct, total, ), "true_positive": true_positive, "true_negative": true_negative, "false_positive": false_positive, "false_negative": false_negative, "precision": precision, "recall": recall, "f1": f1, "result_count_correct": result_count_correct, "result_count_accuracy": safe_divide( result_count_correct, total, ), "positive_records": positive_total, "entity_type_correct": entity_type_correct, "entity_type_accuracy": safe_divide( entity_type_correct, positive_total, ), "entity_text_correct": entity_text_correct, "entity_text_accuracy": safe_divide( entity_text_correct, positive_total, ), "evidence_correct": evidence_correct, "evidence_accuracy": safe_divide( evidence_correct, positive_total, ), } summary_path.write_text( json.dumps(summary, indent=2) + "\n", encoding="utf-8", ) if output_dir.exists(): shutil.rmtree(output_dir) temporary_dir.rename(output_dir) print() print("Development evaluation completed.") print(f"Total records: {total}") print( "Valid JSON: " f"{valid_json_count}/{total} " f"({summary['valid_json_rate']:.4%})" ) print( "Valid schema: " f"{valid_schema_count}/{total} " f"({summary['valid_schema_rate']:.4%})" ) print( "Classification accuracy:" f" {summary['classification_accuracy']:.4%}" ) print(f"Precision: {precision:.4%}") print(f"Recall: {recall:.4%}") print(f"F1: {f1:.4%}") print( "Result count accuracy: " f"{summary['result_count_accuracy']:.4%}" ) print( "Entity type accuracy: " f"{summary['entity_type_accuracy']:.4%}" ) print( "Entity text accuracy: " f"{summary['entity_text_accuracy']:.4%}" ) print( "Evidence accuracy: " f"{summary['evidence_accuracy']:.4%}" ) print(f"Predictions: {output_dir / 'predictions.jsonl'}") print(f"Summary: {output_dir / 'summary.json'}") if __name__ == "__main__": main()