{
 "license": "CC-BY-4.0",
 "attribution": "Fide AI, Agentic Cyber Explorer",
 "url": "https://agentic-cyber-explorer.pages.dev/findings/public-ctf-benchmarks-contaminated/",
 "asOf": "2026-09-26",
 "id": "public-ctf-benchmarks-contaminated",
 "claim": "Scores on public CTF benchmarks can be inflated when agents find published solutions, and static benchmarks lose validity as their flaws are patched.",
 "evidenceKind": "measured",
 "scope": "One contamination measurement (CTFusion, on NYU CTF Bench) and one position paper that argues, without new measurements, that static benchmarks go stale.",
 "topics": [
  "eval-validity"
 ],
 "atlas": [],
 "evidence": [
  {
   "event": "ctfusion-live-ctf-contamination-2026",
   "note": "On NYU CTF Bench, web search raised the D-CIPHER agent's solve rate from 12.59% to 24.07%, with 71 logged cases of copied flags or retrieved writeups."
  },
  {
   "event": "measuring-security-without-fooling-ourselves-2026",
   "note": "Argues, without new measurements, that static benchmarks go stale as vulnerabilities are patched and fixes are memorized."
  }
 ],
 "relations": [],
 "statusHistory": [
  {
   "status": "reported",
   "on": "2026-05-12",
   "why": "CTFusion shows web-searching agents inflate public CTF scores.",
   "event": "ctfusion-live-ctf-contamination-2026",
   "kind": "evidence"
  },
  {
   "status": "corroborated",
   "on": "2026-05-21",
   "why": "A separate paper argues static cyber benchmarks decay.",
   "event": "measuring-security-without-fooling-ourselves-2026",
   "kind": "evidence"
  },
  {
   "status": "reported",
   "on": "2026-09-25",
   "why": "Correction: the 2026-05-21 paper argues that benchmarks go stale but does not measure or independently test contamination, so the measured part of this claim rests on CTFusion alone.",
   "event": "ctfusion-live-ctf-contamination-2026",
   "kind": "correction"
  }
 ],
 "halfLifeDays": 365,
 "wouldChange": "Contamination-resistant benchmarks becoming standard.",
 "fideQuestions": [
  "FID-075"
 ],
 "methods": [
  "ctf-benchmarks"
 ],
 "review": "assistant-drafted",
 "addedOn": "2026-09-25"
}