Models-Phi-8B/scripts/rescore_evaluation.py

216 lines
6.2 KiB
Python

from __future__ import annotations
import json
from pathlib import Path
from evaluate import safe_divide, validate_schema
PROJECT_ROOT = Path(__file__).resolve().parent.parent
OUTPUT_DIR = PROJECT_ROOT / "outputs" / "development-evaluation"
PREDICTIONS_PATH = OUTPUT_DIR / "predictions.jsonl"
SUMMARY_PATH = OUTPUT_DIR / "summary.json"
def main() -> None:
records = []
valid_json_count = 0
valid_schema_count = 0
classification_correct = 0
result_count_correct = 0
true_positive = 0
true_negative = 0
false_positive = 0
false_negative = 0
positive_total = 0
entity_type_correct = 0
entity_text_correct = 0
evidence_correct = 0
decoder = json.JSONDecoder()
with PREDICTIONS_PATH.open("r", encoding="utf-8") as file:
for line in file:
record = json.loads(line)
expected = record["expected"]
raw_output = record["raw_output"]
parsed = None
json_valid = False
schema_valid = False
try:
parsed, _ = decoder.raw_decode(raw_output.lstrip())
json_valid = True
valid_json_count += 1
except json.JSONDecodeError:
pass
if json_valid and validate_schema(parsed):
schema_valid = True
valid_schema_count += 1
gold_phi = expected["is_medical_phi"]
if gold_phi:
positive_total += 1
if schema_valid:
predicted_phi = parsed["is_medical_phi"]
if predicted_phi == gold_phi:
classification_correct += 1
if gold_phi and predicted_phi:
true_positive += 1
elif not gold_phi and not predicted_phi:
true_negative += 1
elif not gold_phi and predicted_phi:
false_positive += 1
else:
false_negative += 1
expected_results = expected["results"]
predicted_results = parsed["results"]
if len(predicted_results) == len(expected_results):
result_count_correct += 1
if gold_phi and expected_results and predicted_results:
expected_result = expected_results[0]
predicted_result = predicted_results[0]
if (
predicted_result["entity_type"]
== expected_result["entity_type"]
):
entity_type_correct += 1
if (
predicted_result["text"]
== expected_result["text"]
):
entity_text_correct += 1
if (
predicted_result["evidence"]
== expected_result["evidence"]
):
evidence_correct += 1
record["parsed_output"] = parsed
record["json_valid"] = json_valid
record["schema_valid"] = schema_valid
records.append(record)
total = len(records)
precision = safe_divide(
true_positive,
true_positive + false_positive,
)
recall = safe_divide(
true_positive,
true_positive + false_negative,
)
f1 = safe_divide(
2 * precision * recall,
precision + recall,
)
summary = {
"total_records": total,
"valid_json": valid_json_count,
"valid_json_rate": safe_divide(valid_json_count, total),
"valid_schema": valid_schema_count,
"valid_schema_rate": safe_divide(valid_schema_count, total),
"classification_correct": classification_correct,
"classification_accuracy": safe_divide(
classification_correct,
total,
),
"true_positive": true_positive,
"true_negative": true_negative,
"false_positive": false_positive,
"false_negative": false_negative,
"precision": precision,
"recall": recall,
"f1": f1,
"result_count_correct": result_count_correct,
"result_count_accuracy": safe_divide(
result_count_correct,
total,
),
"positive_records": positive_total,
"entity_type_correct": entity_type_correct,
"entity_type_accuracy": safe_divide(
entity_type_correct,
positive_total,
),
"entity_text_correct": entity_text_correct,
"entity_text_accuracy": safe_divide(
entity_text_correct,
positive_total,
),
"evidence_correct": evidence_correct,
"evidence_accuracy": safe_divide(
evidence_correct,
positive_total,
),
}
temporary_path = PREDICTIONS_PATH.with_suffix(".jsonl.tmp")
with temporary_path.open("w", encoding="utf-8", newline="\n") as file:
for record in records:
file.write(
json.dumps(
record,
ensure_ascii=False,
separators=(",", ":"),
)
+ "\n"
)
temporary_path.replace(PREDICTIONS_PATH)
SUMMARY_PATH.write_text(
json.dumps(summary, indent=2) + "\n",
encoding="utf-8",
)
print("Development evaluation rescored.")
print(f"Total records: {total}")
print(f"Valid JSON: {valid_json_count}/{total}")
print(f"Valid schema: {valid_schema_count}/{total}")
print(
"Classification accuracy:"
f" {summary['classification_accuracy']:.4%}"
)
print(f"Precision: {precision:.4%}")
print(f"Recall: {recall:.4%}")
print(f"F1: {f1:.4%}")
print(
"Result count accuracy: "
f"{summary['result_count_accuracy']:.4%}"
)
print(
"Entity type accuracy: "
f"{summary['entity_type_accuracy']:.4%}"
)
print(
"Entity text accuracy: "
f"{summary['entity_text_accuracy']:.4%}"
)
print(
"Evidence accuracy: "
f"{summary['evidence_accuracy']:.4%}"
)
if __name__ == "__main__":
main()