{
 "license": "CC-BY-4.0",
 "attribution": "Fide AI, Agentic Cyber Explorer",
 "url": "https://agentic-cyber-explorer.pages.dev/events/ctfusion-live-ctf-contamination-2026/",
 "asOf": "2026-09-26",
 "id": "ctfusion-live-ctf-contamination-2026",
 "date": "2026-05-12",
 "datePrecision": "day",
 "title": "CTFusion uses live CTF events to counter contamination and cheating in cyber agent benchmarks",
 "lane": "defense",
 "kind": "benchmark",
 "summary": "Lee, Bae and Yun show that existing CTF benchmarks can be solved by retrieving published writeups when agents have web search, and propose CTFusion, which evaluates agents on live CTF competitions through an MCP server on the CTFd platform. They test 3 LLMs and 2 agent designs across 5 live CTF events.",
 "whyItMatters": "Static CTF benchmarks underpin many cyber capability claims, and this work demonstrates a concrete contamination path.",
 "actors": [],
 "topics": [
  "eval-validity",
  "capability-evaluation"
 ],
 "atlas": [
  "eval-environment"
 ],
 "artifacts": [
  "ctfusion",
  "claude-sonnet",
  "gemini",
  "gpt-4-family"
 ],
 "sources": [
  {
   "url": "https://arxiv.org/abs/2605.11504",
   "publisher": "arXiv",
   "title": "CTFusion: A CTF-based Benchmark for LLM Agent Evaluation",
   "date": "2026-05-12",
   "type": "primary",
   "accessed": "2026-09-25"
  }
 ],
 "keyFacts": [
  {
   "fact": "Adding web search to the D-CIPHER agent raised its NYU CTF Bench solve rate from 12.59% to 24.07%; the authors traced several submitted flags to public solutions (63 flag-copying and 8 write-up-search cases).",
   "locator": "Introduction; Section 3"
  },
  {
   "fact": "Across 3 LLMs (GPT-4.1, Claude 3.5 Sonnet, Gemini 2.5 Flash) and 2 agents (EnIGMA, D-CIPHER), success was 14.4% on NYU CTF Bench vs 6.3% on 5 live CTFs; the authors say contamination may explain part of the gap.",
   "locator": "Introduction; evaluation section"
  }
 ],
 "significance": 3,
 "fideQuestions": [
  "FID-075"
 ],
 "methods": [
  "adaptive-red-teaming",
  "ctf-benchmarks",
  "evaluation-gaming"
 ],
 "review": "assistant-drafted",
 "addedOn": "2026-09-25"
}