from __future__ import annotations import json from pathlib import Path from evaluate import safe_divide, validate_schema PROJECT_ROOT = Path(__file__).resolve().parent.parent OUTPUT_DIR = PROJECT_ROOT / "outputs" / "development-evaluation" PREDICTIONS_PATH = OUTPUT_DIR / "predictions.jsonl" SUMMARY_PATH = OUTPUT_DIR / "summary.json" def main() -> None: records = [] valid_json_count = 0 valid_schema_count = 0 classification_correct = 0 result_count_correct = 0 true_positive = 0 true_negative = 0 false_positive = 0 false_negative = 0 positive_total = 0 entity_type_correct = 0 entity_text_correct = 0 evidence_correct = 0 decoder = json.JSONDecoder() with PREDICTIONS_PATH.open("r", encoding="utf-8") as file: for line in file: record = json.loads(line) expected = record["expected"] raw_output = record["raw_output"] parsed = None json_valid = False schema_valid = False try: parsed, _ = decoder.raw_decode(raw_output.lstrip()) json_valid = True valid_json_count += 1 except json.JSONDecodeError: pass if json_valid and validate_schema(parsed): schema_valid = True valid_schema_count += 1 gold_phi = expected["is_medical_phi"] if gold_phi: positive_total += 1 if schema_valid: predicted_phi = parsed["is_medical_phi"] if predicted_phi == gold_phi: classification_correct += 1 if gold_phi and predicted_phi: true_positive += 1 elif not gold_phi and not predicted_phi: true_negative += 1 elif not gold_phi and predicted_phi: false_positive += 1 else: false_negative += 1 expected_results = expected["results"] predicted_results = parsed["results"] if len(predicted_results) == len(expected_results): result_count_correct += 1 if gold_phi and expected_results and predicted_results: expected_result = expected_results[0] predicted_result = predicted_results[0] if ( predicted_result["entity_type"] == expected_result["entity_type"] ): entity_type_correct += 1 if ( predicted_result["text"] == expected_result["text"] ): entity_text_correct += 1 if ( predicted_result["evidence"] == expected_result["evidence"] ): evidence_correct += 1 record["parsed_output"] = parsed record["json_valid"] = json_valid record["schema_valid"] = schema_valid records.append(record) total = len(records) precision = safe_divide( true_positive, true_positive + false_positive, ) recall = safe_divide( true_positive, true_positive + false_negative, ) f1 = safe_divide( 2 * precision * recall, precision + recall, ) summary = { "total_records": total, "valid_json": valid_json_count, "valid_json_rate": safe_divide(valid_json_count, total), "valid_schema": valid_schema_count, "valid_schema_rate": safe_divide(valid_schema_count, total), "classification_correct": classification_correct, "classification_accuracy": safe_divide( classification_correct, total, ), "true_positive": true_positive, "true_negative": true_negative, "false_positive": false_positive, "false_negative": false_negative, "precision": precision, "recall": recall, "f1": f1, "result_count_correct": result_count_correct, "result_count_accuracy": safe_divide( result_count_correct, total, ), "positive_records": positive_total, "entity_type_correct": entity_type_correct, "entity_type_accuracy": safe_divide( entity_type_correct, positive_total, ), "entity_text_correct": entity_text_correct, "entity_text_accuracy": safe_divide( entity_text_correct, positive_total, ), "evidence_correct": evidence_correct, "evidence_accuracy": safe_divide( evidence_correct, positive_total, ), } temporary_path = PREDICTIONS_PATH.with_suffix(".jsonl.tmp") with temporary_path.open("w", encoding="utf-8", newline="\n") as file: for record in records: file.write( json.dumps( record, ensure_ascii=False, separators=(",", ":"), ) + "\n" ) temporary_path.replace(PREDICTIONS_PATH) SUMMARY_PATH.write_text( json.dumps(summary, indent=2) + "\n", encoding="utf-8", ) print("Development evaluation rescored.") print(f"Total records: {total}") print(f"Valid JSON: {valid_json_count}/{total}") print(f"Valid schema: {valid_schema_count}/{total}") print( "Classification accuracy:" f" {summary['classification_accuracy']:.4%}" ) print(f"Precision: {precision:.4%}") print(f"Recall: {recall:.4%}") print(f"F1: {f1:.4%}") print( "Result count accuracy: " f"{summary['result_count_accuracy']:.4%}" ) print( "Entity type accuracy: " f"{summary['entity_type_accuracy']:.4%}" ) print( "Entity text accuracy: " f"{summary['entity_text_accuracy']:.4%}" ) print( "Evidence accuracy: " f"{summary['evidence_accuracy']:.4%}" ) if __name__ == "__main__": main()