{
 "license": "CC-BY-4.0",
 "attribution": "Fide AI, Agentic Cyber Explorer",
 "url": "https://agentic-cyber-explorer.pages.dev/events/cwe-bench-v2-public-repair-evaluation-2026/",
 "asOf": "2026-11-17",
 "id": "cwe-bench-v2-public-repair-evaluation-2026",
 "date": "2026-11-03",
 "datePrecision": "day",
 "title": "CWE-bench v2 releases inspectable security-repair tasks alongside held-out tests",
 "lane": "defense",
 "kind": "benchmark",
 "summary": "Collinear released CWE-bench v2 with 100 public repository security-repair tasks covering 91 distinct CWEs, alongside held-out evaluation. Tasks check security and existing functionality, with programmatic graders and human review. Public-set results and the separate combined leaderboard use different denominators, and neither is a direct continuation of v1 scores.",
 "whyItMatters": "It improves inspection of defensive agent evaluations while keeping benchmark version, task selection and execution harness visible.",
 "actors": [
  "collinear"
 ],
 "topics": [
  "vulnerability-repair",
  "capability-evaluation",
  "eval-validity"
 ],
 "atlas": [],
 "artifacts": [
  "cwe-bench"
 ],
 "sources": [
  {
   "url": "https://huggingface.co/blog/cwe-bench",
   "publisher": "Collinear / Hugging Face",
   "title": "CWE-bench v2: More coverage, quality, and transparency",
   "date": "2026-11-03",
   "type": "primary",
   "accessed": "2026-11-15"
  },
  {
   "url": "https://cwe-bench.com/",
   "publisher": "Collinear",
   "title": "CWE-bench benchmark and leaderboard",
   "type": "primary",
   "accessed": "2026-11-15"
  }
 ],
 "keyFacts": [
  {
   "fact": "The v2 public set has 100 tasks covering 91 CWEs. Access requires accepting the dataset’s conditions; the grader and harness are available for inspection.",
   "locator": "Release: introduction; Run it from the Hub"
  },
  {
   "fact": "Collinear describes LLM-assisted verifier curation and manual review of every task, checking security behavior and preservation of original functionality. These are benchmark checks, not guarantees for deployed patches.",
   "locator": "Release: quality checklist"
  },
  {
   "fact": "The separate CWE-bench site describes 150 held-out tasks in addition to the 100 public tasks, for 250 tasks and 185 distinct CWEs overall. Public-set and combined leaderboard figures must retain their task-set labels.",
   "locator": "CWE-bench site: benchmark overview"
  },
  {
   "fact": "Scores depend on version, agent harness and attempt budget. The public v2 set and combined leaderboard should not replace the earlier private v1 record’s results.",
   "locator": "Release: Results; CWE-bench site: methodology"
  }
 ],
 "significance": 4,
 "fideQuestions": [
  "FID-075",
  "FID-088"
 ],
 "methods": [
  "automated-patching",
  "ctf-benchmarks",
  "patch-verification"
 ],
 "review": "assistant-drafted",
 "addedOn": "2026-11-15",
 "revisions": []
}