{
 "license": "CC-BY-4.0",
 "attribution": "Fide AI, Agentic Cyber Explorer",
 "url": "https://agentic-cyber-explorer.pages.dev/events/sec-bench-2025/",
 "asOf": "2026-09-26",
 "id": "sec-bench-2025",
 "date": "2025-06-13",
 "datePrecision": "day",
 "title": "SEC-bench automatically builds real vulnerability tasks and finds agents patch at most 34%",
 "lane": "defense",
 "kind": "benchmark",
 "summary": "SEC-bench uses multi-agent scaffolding to construct reproducible vulnerability instances with test environments and validated patches from real projects, at about $0.87 per instance. The authors report that LLM agents reached at most 18.0% on proof-of-concept generation and 34.0% on vulnerability patching.",
 "whyItMatters": "It offers a cheaper route to fresh vulnerability benchmarks and shows low agent patching rates even with call-stack hints and a build-and-PoC check.",
 "actors": [
  "uiuc",
  "purdue-university"
 ],
 "topics": [
  "vulnerability-repair",
  "capability-evaluation"
 ],
 "atlas": [
  "eval-environment"
 ],
 "artifacts": [
  "sec-bench",
  "claude-sonnet"
 ],
 "sources": [
  {
   "url": "https://arxiv.org/abs/2506.11791",
   "publisher": "arXiv",
   "title": "SEC-bench: Automated Benchmarking of LLM Agents on Real-World Software Security Tasks",
   "date": "2025-06-13",
   "type": "primary",
   "accessed": "2026-09-25"
  }
 ],
 "keyFacts": [
  {
   "fact": "Dataset construction cost about $0.87 per instance.",
   "locator": "Abstract"
  },
  {
   "fact": "On the full 200-instance dataset with Claude 3.7 Sonnet, the best scaffold (OpenHands) reached 18.0% on PoC generation and 34.0% on vulnerability patching; a patch counts if the project builds and the original PoC no longer triggers.",
   "locator": "Sections 2 and 3.2"
  }
 ],
 "significance": 2,
 "fideQuestions": [
  "FID-075",
  "FID-088"
 ],
 "methods": [
  "automated-patching"
 ],
 "review": "assistant-drafted",
 "addedOn": "2026-09-25"
}