216 lines
6.2 KiB
Python
216 lines
6.2 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from evaluate import safe_divide, validate_schema
|
|
|
|
|
|
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
|
OUTPUT_DIR = PROJECT_ROOT / "outputs" / "development-evaluation"
|
|
PREDICTIONS_PATH = OUTPUT_DIR / "predictions.jsonl"
|
|
SUMMARY_PATH = OUTPUT_DIR / "summary.json"
|
|
|
|
|
|
def main() -> None:
|
|
records = []
|
|
|
|
valid_json_count = 0
|
|
valid_schema_count = 0
|
|
classification_correct = 0
|
|
result_count_correct = 0
|
|
|
|
true_positive = 0
|
|
true_negative = 0
|
|
false_positive = 0
|
|
false_negative = 0
|
|
|
|
positive_total = 0
|
|
entity_type_correct = 0
|
|
entity_text_correct = 0
|
|
evidence_correct = 0
|
|
|
|
decoder = json.JSONDecoder()
|
|
|
|
with PREDICTIONS_PATH.open("r", encoding="utf-8") as file:
|
|
for line in file:
|
|
record = json.loads(line)
|
|
expected = record["expected"]
|
|
raw_output = record["raw_output"]
|
|
|
|
parsed = None
|
|
json_valid = False
|
|
schema_valid = False
|
|
|
|
try:
|
|
parsed, _ = decoder.raw_decode(raw_output.lstrip())
|
|
json_valid = True
|
|
valid_json_count += 1
|
|
except json.JSONDecodeError:
|
|
pass
|
|
|
|
if json_valid and validate_schema(parsed):
|
|
schema_valid = True
|
|
valid_schema_count += 1
|
|
|
|
gold_phi = expected["is_medical_phi"]
|
|
|
|
if gold_phi:
|
|
positive_total += 1
|
|
|
|
if schema_valid:
|
|
predicted_phi = parsed["is_medical_phi"]
|
|
|
|
if predicted_phi == gold_phi:
|
|
classification_correct += 1
|
|
|
|
if gold_phi and predicted_phi:
|
|
true_positive += 1
|
|
elif not gold_phi and not predicted_phi:
|
|
true_negative += 1
|
|
elif not gold_phi and predicted_phi:
|
|
false_positive += 1
|
|
else:
|
|
false_negative += 1
|
|
|
|
expected_results = expected["results"]
|
|
predicted_results = parsed["results"]
|
|
|
|
if len(predicted_results) == len(expected_results):
|
|
result_count_correct += 1
|
|
|
|
if gold_phi and expected_results and predicted_results:
|
|
expected_result = expected_results[0]
|
|
predicted_result = predicted_results[0]
|
|
|
|
if (
|
|
predicted_result["entity_type"]
|
|
== expected_result["entity_type"]
|
|
):
|
|
entity_type_correct += 1
|
|
|
|
if (
|
|
predicted_result["text"]
|
|
== expected_result["text"]
|
|
):
|
|
entity_text_correct += 1
|
|
|
|
if (
|
|
predicted_result["evidence"]
|
|
== expected_result["evidence"]
|
|
):
|
|
evidence_correct += 1
|
|
|
|
record["parsed_output"] = parsed
|
|
record["json_valid"] = json_valid
|
|
record["schema_valid"] = schema_valid
|
|
records.append(record)
|
|
|
|
total = len(records)
|
|
|
|
precision = safe_divide(
|
|
true_positive,
|
|
true_positive + false_positive,
|
|
)
|
|
recall = safe_divide(
|
|
true_positive,
|
|
true_positive + false_negative,
|
|
)
|
|
f1 = safe_divide(
|
|
2 * precision * recall,
|
|
precision + recall,
|
|
)
|
|
|
|
summary = {
|
|
"total_records": total,
|
|
"valid_json": valid_json_count,
|
|
"valid_json_rate": safe_divide(valid_json_count, total),
|
|
"valid_schema": valid_schema_count,
|
|
"valid_schema_rate": safe_divide(valid_schema_count, total),
|
|
"classification_correct": classification_correct,
|
|
"classification_accuracy": safe_divide(
|
|
classification_correct,
|
|
total,
|
|
),
|
|
"true_positive": true_positive,
|
|
"true_negative": true_negative,
|
|
"false_positive": false_positive,
|
|
"false_negative": false_negative,
|
|
"precision": precision,
|
|
"recall": recall,
|
|
"f1": f1,
|
|
"result_count_correct": result_count_correct,
|
|
"result_count_accuracy": safe_divide(
|
|
result_count_correct,
|
|
total,
|
|
),
|
|
"positive_records": positive_total,
|
|
"entity_type_correct": entity_type_correct,
|
|
"entity_type_accuracy": safe_divide(
|
|
entity_type_correct,
|
|
positive_total,
|
|
),
|
|
"entity_text_correct": entity_text_correct,
|
|
"entity_text_accuracy": safe_divide(
|
|
entity_text_correct,
|
|
positive_total,
|
|
),
|
|
"evidence_correct": evidence_correct,
|
|
"evidence_accuracy": safe_divide(
|
|
evidence_correct,
|
|
positive_total,
|
|
),
|
|
}
|
|
|
|
temporary_path = PREDICTIONS_PATH.with_suffix(".jsonl.tmp")
|
|
|
|
with temporary_path.open("w", encoding="utf-8", newline="\n") as file:
|
|
for record in records:
|
|
file.write(
|
|
json.dumps(
|
|
record,
|
|
ensure_ascii=False,
|
|
separators=(",", ":"),
|
|
)
|
|
+ "\n"
|
|
)
|
|
|
|
temporary_path.replace(PREDICTIONS_PATH)
|
|
|
|
SUMMARY_PATH.write_text(
|
|
json.dumps(summary, indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
|
|
print("Development evaluation rescored.")
|
|
print(f"Total records: {total}")
|
|
print(f"Valid JSON: {valid_json_count}/{total}")
|
|
print(f"Valid schema: {valid_schema_count}/{total}")
|
|
print(
|
|
"Classification accuracy:"
|
|
f" {summary['classification_accuracy']:.4%}"
|
|
)
|
|
print(f"Precision: {precision:.4%}")
|
|
print(f"Recall: {recall:.4%}")
|
|
print(f"F1: {f1:.4%}")
|
|
print(
|
|
"Result count accuracy: "
|
|
f"{summary['result_count_accuracy']:.4%}"
|
|
)
|
|
print(
|
|
"Entity type accuracy: "
|
|
f"{summary['entity_type_accuracy']:.4%}"
|
|
)
|
|
print(
|
|
"Entity text accuracy: "
|
|
f"{summary['entity_text_accuracy']:.4%}"
|
|
)
|
|
print(
|
|
"Evidence accuracy: "
|
|
f"{summary['evidence_accuracy']:.4%}"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|