{
 "license": "CC-BY-4.0",
 "attribution": "Fide AI, Agentic Cyber Explorer",
 "url": "https://agentic-cyber-explorer.pages.dev/findings/crash-checks-overstate-patch-success/",
 "asOf": "2026-09-26",
 "id": "crash-checks-overstate-patch-success",
 "claim": "Checking only that the original crash no longer reproduces overstates how often AI-generated patches actually fix the vulnerability.",
 "evidenceKind": "measured",
 "scope": "Two analyses of research and competition patches. PatchBench's 1.83x is measured on tasks selected so that the true fix lies outside the crash stack, so it is not a base rate.",
 "topics": [
  "vulnerability-repair",
  "eval-validity"
 ],
 "atlas": [],
 "evidence": [
  {
   "event": "aixcc-sok-competition-lessons-2026",
   "note": "Manual review found 37.7% and 45.6% of baseline-agent patches that passed all automatic checks semantically incorrect."
  },
  {
   "event": "patchbench-vulnerability-patching-validity-2026",
   "note": "Proof-of-concept-only checks inflated success 1.83 times on PatchBench's tasks."
  }
 ],
 "relations": [
  {
   "type": "qualifies",
   "target": "aixcc-systems-patched-most-found-bugs",
   "note": "The AIxCC teams' agents lose much of their solve rate under stronger validation on PatchBench tasks; the competition's counted patches were not re-validated."
  }
 ],
 "statusHistory": [
  {
   "status": "reported",
   "on": "2026-02-07",
   "why": "AIxCC review finds semantically wrong top patches.",
   "event": "aixcc-sok-competition-lessons-2026",
   "kind": "evidence"
  },
  {
   "status": "corroborated",
   "on": "2026-09-03",
   "why": "PatchBench measures the inflation directly.",
   "event": "patchbench-vulnerability-patching-validity-2026",
   "kind": "evidence"
  },
  {
   "status": "corroborated",
   "on": "2026-09-25",
   "why": "Correction: the review paper's support is its manual review of baseline agents (38-46% of fully validated patches semantically wrong), not the competition-scored accuracy figures.",
   "event": "aixcc-sok-competition-lessons-2026",
   "kind": "correction"
  }
 ],
 "halfLifeDays": 540,
 "wouldChange": "Verification practice that closes the gap in deployed repair tools.",
 "fideQuestions": [
  "FID-088"
 ],
 "methods": [
  "automated-patching",
  "patch-verification"
 ],
 "review": "assistant-drafted",
 "addedOn": "2026-09-25"
}