494 lines
13 KiB
Python
494 lines
13 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import shutil
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import torch
|
|
from peft import PeftModel
|
|
from tqdm import tqdm
|
|
|
|
from json_stopping import JsonObjectStoppingCriteria
|
|
from transformers import (
|
|
StoppingCriteriaList,
|
|
AutoModelForCausalLM,
|
|
AutoTokenizer,
|
|
BitsAndBytesConfig,
|
|
)
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
CONFIG_PATH = PROJECT_ROOT / "config" / "pipeline.json"
|
|
CHAT_TEMPLATE_PATH = PROJECT_ROOT / "config" / "chat_template.jinja"
|
|
|
|
ALLOWED_ENTITY_TYPES = {
|
|
"MEDICAL_ALLERGY",
|
|
"MEDICAL_DIAGNOSIS",
|
|
"MEDICAL_FAMILY_HISTORY",
|
|
"MEDICAL_MEDICATION",
|
|
"MEDICAL_MENTAL_HEALTH",
|
|
"MEDICAL_OTHER",
|
|
"MEDICAL_PHYSICAL_HEALTH",
|
|
"MEDICAL_PROCEDURE",
|
|
"MEDICAL_SUBSTANCE_USE",
|
|
"MEDICAL_SYMPTOM",
|
|
"MEDICAL_TEST",
|
|
"MEDICAL_TREATMENT",
|
|
}
|
|
|
|
|
|
def read_jsonl(path: Path) -> list[dict[str, Any]]:
|
|
records: list[dict[str, Any]] = []
|
|
|
|
with path.open("r", encoding="utf-8") as file:
|
|
for line in file:
|
|
if line.strip():
|
|
records.append(json.loads(line))
|
|
|
|
return records
|
|
|
|
|
|
def is_number(value: Any) -> bool:
|
|
return (
|
|
isinstance(value, (int, float))
|
|
and not isinstance(value, bool)
|
|
)
|
|
|
|
|
|
def valid_score(value: Any) -> bool:
|
|
return is_number(value) and 0.0 <= float(value) <= 1.0
|
|
|
|
|
|
def validate_schema(value: Any) -> bool:
|
|
if not isinstance(value, dict):
|
|
return False
|
|
|
|
if set(value) != {
|
|
"is_medical_phi",
|
|
"score",
|
|
"reason",
|
|
"results",
|
|
}:
|
|
return False
|
|
|
|
if not isinstance(value["is_medical_phi"], bool):
|
|
return False
|
|
|
|
if not valid_score(value["score"]):
|
|
return False
|
|
|
|
if not isinstance(value["reason"], str):
|
|
return False
|
|
|
|
if not isinstance(value["results"], list):
|
|
return False
|
|
|
|
for result in value["results"]:
|
|
if not isinstance(result, dict):
|
|
return False
|
|
|
|
if set(result) != {
|
|
"entity_type",
|
|
"text",
|
|
"score",
|
|
"evidence",
|
|
}:
|
|
return False
|
|
|
|
if result["entity_type"] not in ALLOWED_ENTITY_TYPES:
|
|
return False
|
|
|
|
if not isinstance(result["text"], str):
|
|
return False
|
|
|
|
if not valid_score(result["score"]):
|
|
return False
|
|
|
|
if not isinstance(result["evidence"], str):
|
|
return False
|
|
|
|
return True
|
|
|
|
|
|
def safe_divide(numerator: int, denominator: int) -> float:
|
|
if denominator == 0:
|
|
return 0.0
|
|
|
|
return numerator / denominator
|
|
|
|
|
|
def main() -> None:
|
|
config = json.loads(CONFIG_PATH.read_text(encoding="utf-8"))
|
|
|
|
model_path = config["model"]["base_model"]
|
|
max_length = int(config["model"]["max_sequence_length"])
|
|
|
|
adapter_path = (
|
|
PROJECT_ROOT
|
|
/ config["training"]["full"]["output_dir"]
|
|
)
|
|
|
|
development_path = (
|
|
PROJECT_ROOT
|
|
/ config["paths"]["development"]
|
|
)
|
|
|
|
output_dir = PROJECT_ROOT / "outputs" / "development-evaluation"
|
|
temporary_dir = PROJECT_ROOT / "outputs" / "development-evaluation.tmp"
|
|
|
|
if temporary_dir.exists():
|
|
shutil.rmtree(temporary_dir)
|
|
|
|
temporary_dir.mkdir(parents=True)
|
|
|
|
predictions_path = temporary_dir / "predictions.jsonl"
|
|
summary_path = temporary_dir / "summary.json"
|
|
|
|
records = read_jsonl(development_path)
|
|
|
|
print(f"Development records: {len(records)}")
|
|
print("Loading tokenizer...")
|
|
|
|
tokenizer = AutoTokenizer.from_pretrained(
|
|
model_path,
|
|
use_fast=True,
|
|
)
|
|
|
|
tokenizer.chat_template = CHAT_TEMPLATE_PATH.read_text(
|
|
encoding="utf-8"
|
|
)
|
|
|
|
if tokenizer.pad_token_id is None:
|
|
tokenizer.pad_token = tokenizer.eos_token
|
|
|
|
tokenizer.padding_side = "left"
|
|
|
|
quantization_config = BitsAndBytesConfig(
|
|
load_in_4bit=True,
|
|
bnb_4bit_quant_type="nf4",
|
|
bnb_4bit_use_double_quant=True,
|
|
bnb_4bit_compute_dtype=torch.bfloat16,
|
|
)
|
|
|
|
print("Loading base model in 4-bit NF4...")
|
|
|
|
base_model = AutoModelForCausalLM.from_pretrained(
|
|
model_path,
|
|
quantization_config=quantization_config,
|
|
dtype=torch.bfloat16,
|
|
device_map={"": 0},
|
|
low_cpu_mem_usage=True,
|
|
)
|
|
|
|
print(f"Loading adapter: {adapter_path}")
|
|
|
|
model = PeftModel.from_pretrained(
|
|
base_model,
|
|
adapter_path,
|
|
)
|
|
|
|
model.eval()
|
|
model.config.use_cache = True
|
|
|
|
eos_token_ids = [tokenizer.eos_token_id]
|
|
eot_token_id = tokenizer.convert_tokens_to_ids("<|eot_id|>")
|
|
|
|
if (
|
|
isinstance(eot_token_id, int)
|
|
and eot_token_id >= 0
|
|
and eot_token_id not in eos_token_ids
|
|
):
|
|
eos_token_ids.append(eot_token_id)
|
|
|
|
total = len(records)
|
|
valid_json_count = 0
|
|
valid_schema_count = 0
|
|
classification_correct = 0
|
|
result_count_correct = 0
|
|
|
|
true_positive = 0
|
|
true_negative = 0
|
|
false_positive = 0
|
|
false_negative = 0
|
|
|
|
positive_total = 0
|
|
entity_type_correct = 0
|
|
entity_text_correct = 0
|
|
evidence_correct = 0
|
|
|
|
batch_size = 8
|
|
max_new_tokens = 256
|
|
|
|
print("Running deterministic development evaluation...")
|
|
|
|
with predictions_path.open(
|
|
"w",
|
|
encoding="utf-8",
|
|
newline="\n",
|
|
) as prediction_file:
|
|
for start in tqdm(
|
|
range(0, total, batch_size),
|
|
desc="Evaluating",
|
|
):
|
|
batch = records[start:start + batch_size]
|
|
|
|
prompts = [
|
|
tokenizer.apply_chat_template(
|
|
record["messages"][:2],
|
|
tokenize=False,
|
|
add_generation_prompt=True,
|
|
)
|
|
for record in batch
|
|
]
|
|
|
|
encoded = tokenizer(
|
|
prompts,
|
|
return_tensors="pt",
|
|
padding=True,
|
|
truncation=True,
|
|
max_length=max_length,
|
|
)
|
|
|
|
encoded = {
|
|
name: tensor.to(model.device)
|
|
for name, tensor in encoded.items()
|
|
}
|
|
|
|
prompt_length = encoded["input_ids"].shape[1]
|
|
|
|
stopping_criteria = StoppingCriteriaList(
|
|
[
|
|
JsonObjectStoppingCriteria(
|
|
tokenizer,
|
|
prompt_length,
|
|
)
|
|
]
|
|
)
|
|
|
|
with torch.inference_mode():
|
|
generated = model.generate(
|
|
**encoded,
|
|
do_sample=False,
|
|
max_new_tokens=max_new_tokens,
|
|
eos_token_id=eos_token_ids,
|
|
pad_token_id=tokenizer.pad_token_id,
|
|
stopping_criteria=stopping_criteria,
|
|
use_cache=True,
|
|
)
|
|
|
|
generated_tokens = generated[:, prompt_length:]
|
|
|
|
raw_outputs = tokenizer.batch_decode(
|
|
generated_tokens,
|
|
skip_special_tokens=True,
|
|
clean_up_tokenization_spaces=False,
|
|
)
|
|
|
|
for record, raw_output in zip(batch, raw_outputs):
|
|
raw_output = raw_output.strip()
|
|
expected = json.loads(
|
|
record["messages"][2]["content"]
|
|
)
|
|
|
|
parsed: Any = None
|
|
json_valid = False
|
|
schema_valid = False
|
|
|
|
try:
|
|
parsed, _ = json.JSONDecoder().raw_decode(
|
|
raw_output.lstrip()
|
|
)
|
|
json_valid = True
|
|
valid_json_count += 1
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
if json_valid and validate_schema(parsed):
|
|
schema_valid = True
|
|
valid_schema_count += 1
|
|
|
|
gold_phi = expected["is_medical_phi"]
|
|
|
|
if gold_phi:
|
|
positive_total += 1
|
|
|
|
if schema_valid:
|
|
predicted_phi = parsed["is_medical_phi"]
|
|
|
|
if predicted_phi == gold_phi:
|
|
classification_correct += 1
|
|
|
|
if gold_phi and predicted_phi:
|
|
true_positive += 1
|
|
elif not gold_phi and not predicted_phi:
|
|
true_negative += 1
|
|
elif not gold_phi and predicted_phi:
|
|
false_positive += 1
|
|
else:
|
|
false_negative += 1
|
|
|
|
expected_results = expected["results"]
|
|
predicted_results = parsed["results"]
|
|
|
|
if len(predicted_results) == len(expected_results):
|
|
result_count_correct += 1
|
|
|
|
if gold_phi and expected_results:
|
|
expected_result = expected_results[0]
|
|
|
|
if predicted_results:
|
|
predicted_result = predicted_results[0]
|
|
|
|
if (
|
|
predicted_result["entity_type"]
|
|
== expected_result["entity_type"]
|
|
):
|
|
entity_type_correct += 1
|
|
|
|
if (
|
|
predicted_result["text"]
|
|
== expected_result["text"]
|
|
):
|
|
entity_text_correct += 1
|
|
|
|
if (
|
|
predicted_result["evidence"]
|
|
== expected_result["evidence"]
|
|
):
|
|
evidence_correct += 1
|
|
|
|
prediction_record = {
|
|
"id": record["id"],
|
|
"input": record["messages"][1]["content"],
|
|
"expected": expected,
|
|
"raw_output": raw_output,
|
|
"parsed_output": parsed,
|
|
"json_valid": json_valid,
|
|
"schema_valid": schema_valid,
|
|
}
|
|
|
|
prediction_file.write(
|
|
json.dumps(
|
|
prediction_record,
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
)
|
|
+ "\n"
|
|
)
|
|
|
|
precision = safe_divide(
|
|
true_positive,
|
|
true_positive + false_positive,
|
|
)
|
|
|
|
recall = safe_divide(
|
|
true_positive,
|
|
true_positive + false_negative,
|
|
)
|
|
|
|
f1 = safe_divide(
|
|
2 * precision * recall,
|
|
precision + recall,
|
|
)
|
|
|
|
summary = {
|
|
"total_records": total,
|
|
"valid_json": valid_json_count,
|
|
"valid_json_rate": safe_divide(
|
|
valid_json_count,
|
|
total,
|
|
),
|
|
"valid_schema": valid_schema_count,
|
|
"valid_schema_rate": safe_divide(
|
|
valid_schema_count,
|
|
total,
|
|
),
|
|
"classification_correct": classification_correct,
|
|
"classification_accuracy": safe_divide(
|
|
classification_correct,
|
|
total,
|
|
),
|
|
"true_positive": true_positive,
|
|
"true_negative": true_negative,
|
|
"false_positive": false_positive,
|
|
"false_negative": false_negative,
|
|
"precision": precision,
|
|
"recall": recall,
|
|
"f1": f1,
|
|
"result_count_correct": result_count_correct,
|
|
"result_count_accuracy": safe_divide(
|
|
result_count_correct,
|
|
total,
|
|
),
|
|
"positive_records": positive_total,
|
|
"entity_type_correct": entity_type_correct,
|
|
"entity_type_accuracy": safe_divide(
|
|
entity_type_correct,
|
|
positive_total,
|
|
),
|
|
"entity_text_correct": entity_text_correct,
|
|
"entity_text_accuracy": safe_divide(
|
|
entity_text_correct,
|
|
positive_total,
|
|
),
|
|
"evidence_correct": evidence_correct,
|
|
"evidence_accuracy": safe_divide(
|
|
evidence_correct,
|
|
positive_total,
|
|
),
|
|
}
|
|
|
|
summary_path.write_text(
|
|
json.dumps(summary, indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
if output_dir.exists():
|
|
shutil.rmtree(output_dir)
|
|
|
|
temporary_dir.rename(output_dir)
|
|
|
|
print()
|
|
print("Development evaluation completed.")
|
|
print(f"Total records: {total}")
|
|
print(
|
|
"Valid JSON: "
|
|
f"{valid_json_count}/{total} "
|
|
f"({summary['valid_json_rate']:.4%})"
|
|
)
|
|
print(
|
|
"Valid schema: "
|
|
f"{valid_schema_count}/{total} "
|
|
f"({summary['valid_schema_rate']:.4%})"
|
|
)
|
|
print(
|
|
"Classification accuracy:"
|
|
f" {summary['classification_accuracy']:.4%}"
|
|
)
|
|
print(f"Precision: {precision:.4%}")
|
|
print(f"Recall: {recall:.4%}")
|
|
print(f"F1: {f1:.4%}")
|
|
print(
|
|
"Result count accuracy: "
|
|
f"{summary['result_count_accuracy']:.4%}"
|
|
)
|
|
print(
|
|
"Entity type accuracy: "
|
|
f"{summary['entity_type_accuracy']:.4%}"
|
|
)
|
|
print(
|
|
"Entity text accuracy: "
|
|
f"{summary['entity_text_accuracy']:.4%}"
|
|
)
|
|
print(
|
|
"Evidence accuracy: "
|
|
f"{summary['evidence_accuracy']:.4%}"
|
|
)
|
|
print(f"Predictions: {output_dir / 'predictions.jsonl'}")
|
|
print(f"Summary: {output_dir / 'summary.json'}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|