{
 "license": "CC-BY-4.0",
 "attribution": "Fide AI, Agentic Cyber Explorer",
 "url": "https://agentic-cyber-explorer.pages.dev/events/patchbench-vulnerability-patching-validity-2026/",
 "asOf": "2026-09-26",
 "id": "patchbench-vulnerability-patching-validity-2026",
 "date": "2026-09-03",
 "datePrecision": "day",
 "title": "PatchBench finds PoC-only checks inflate AI patching success 1.83x and 25% of patches look memorized",
 "lane": "defense",
 "kind": "benchmark",
 "summary": "PatchBench, from the University of Maryland's AI Security Lab, evaluates 11 patching agents, including the top three AIxCC systems, on 213 C/C++ tasks whose true fixes lie outside the crash stack, using vulnerability transplant and code mutation to limit memorization. It finds that accepting a patch because the original proof-of-concept no longer crashes inflates solve rates by 1.83x on average, and that about 25% of agent patches closely resemble historical developer fixes.",
 "whyItMatters": "It directly challenges how AI vulnerability-repair results, including competition results, are validated.",
 "actors": [
  "university-of-maryland"
 ],
 "topics": [
  "vulnerability-repair",
  "eval-validity"
 ],
 "atlas": [
  "eval-environment"
 ],
 "artifacts": [
  "patchbench",
  "aixcc"
 ],
 "sources": [
  {
   "url": "https://arxiv.org/abs/2609.04075",
   "publisher": "arXiv",
   "title": "PatchBench: Evaluating AI Agents for Vulnerability Patching",
   "date": "2026-09-03",
   "type": "primary",
   "accessed": "2026-09-25"
  },
  {
   "url": "https://huggingface.co/datasets/ai-sec-lab/PatchBench",
   "publisher": "Hugging Face (ai-sec-lab)",
   "title": "PatchBench dataset card",
   "type": "primary",
   "accessed": "2026-09-25"
  },
  {
   "url": "https://github.com/ai-sec-lab/PatchBench",
   "publisher": "GitHub (ai-sec-lab)",
   "title": "PatchBench",
   "type": "primary",
   "accessed": "2026-09-25"
  }
 ],
 "keyFacts": [
  {
   "fact": "Across 11 agents, PoC-only validation inflates patching solve rates by 1.83x on average.",
   "locator": "arXiv abstract"
  },
  {
   "fact": "On average 25% of agent patches show substantial similarity to historical developer patches.",
   "locator": "arXiv abstract"
  },
  {
   "fact": "213 tasks from 32 C/C++ GitHub projects; a task counts as solved only if it passes PoC, replay (security), unit test and output-state checks.",
   "locator": "Dataset card; repository README"
  },
  {
   "fact": "Task container images total about 870 GB and additional corpora about 22 GB; metadata is MIT-licensed.",
   "locator": "Repository README; dataset card"
  }
 ],
 "significance": 4,
 "fideQuestions": [
  "FID-075",
  "FID-088"
 ],
 "methods": [
  "automated-patching",
  "sandboxing-egress"
 ],
 "review": "assistant-drafted",
 "addedOn": "2026-09-25"
}