{
 "license": "CC-BY-4.0",
 "attribution": "Fide AI, Agentic Cyber Explorer",
 "url": "https://agentic-cyber-explorer.pages.dev/events/cyberbingym-binary-patch-gap-benchmark-2026/",
 "asOf": "2026-11-17",
 "id": "cyberbingym-binary-patch-gap-benchmark-2026",
 "date": "2026-11-07",
 "datePrecision": "day",
 "title": "CyberBinGym separates binary vulnerability recovery from working exploitation",
 "lane": "capability",
 "kind": "benchmark",
 "summary": "The CyberBinGym preprint evaluates agents given stripped vulnerable and patched binaries for 100 tasks from 53 projects. On the non-hardened baseline, Codex with GPT-5.6 recovers vulnerabilities on 88% of tasks and completes exploitation on 20%. The known-vulnerability patch-gap tests use trusted-access models with safeguards disabled; hardened configurations are evaluated separately.",
 "whyItMatters": "It measures the gap between identifying a bug from a binary patch and producing a verified exploit during delayed deployment.",
 "actors": [],
 "topics": [
  "capability-evaluation",
  "exploit-development",
  "eval-validity"
 ],
 "atlas": [],
 "artifacts": [
  "cyberbingym",
  "gpt-5-family"
 ],
 "sources": [
  {
   "url": "https://arxiv.org/abs/2611.10723v1",
   "publisher": "Wijayadi et al. / arXiv",
   "title": "CyberBinGym: Can AI Agents Turn Binary Patches into Real Exploits?",
   "date": "2026-11-07",
   "type": "primary",
   "accessed": "2026-11-15"
  }
 ],
 "keyFacts": [
  {
   "fact": "The 100 tasks cover 53 projects. Agents receive stripped vulnerable and patched binaries without source code or vulnerability descriptions.",
   "locator": "Abstract; benchmark construction"
  },
  {
   "fact": "Recovery uses differential crash checks; exploitation requires end-to-end flag retrieval and trajectory audits to reject reward hacking. Codex with GPT-5.6 scores 88% and 20%, respectively, on the non-hardened baseline with a nominal 1,000-call budget.",
   "locator": "Evaluation protocol; main results"
  },
  {
   "fact": "Providing a patched binary raises exploitation from 10% to 25% on a matched 20-task subset. This comparison uses that subset rather than the full benchmark denominator.",
   "locator": "Abstract; patched-binary ablation"
  },
  {
   "fact": "In a selected hardened rerun, Codex with GPT-5.6 exploits two of its 20 previously successful tasks. Search variation and service failures mean the aggregate difference does not isolate hardening’s causal effect. Main experiments use trusted-access models with safeguards disabled.",
   "locator": "Hardening experiment; Appendix A.8"
  }
 ],
 "significance": 4,
 "fideQuestions": [
  "FID-075"
 ],
 "methods": [
  "ai-assisted-exploitation",
  "compute-scaled-evaluation",
  "ctf-benchmarks"
 ],
 "review": "assistant-drafted",
 "addedOn": "2026-11-15",
 "revisions": []
}