{
 "license": "CC-BY-4.0",
 "attribution": "Fide AI, Agentic Cyber Explorer",
 "url": "https://agentic-cyber-explorer.pages.dev/events/simbian-cyber-defense-benchmark-threat-hunting-2026/",
 "asOf": "2026-09-26",
 "id": "simbian-cyber-defense-benchmark-threat-hunting-2026",
 "date": "2026-04-21",
 "datePrecision": "day",
 "title": "Threat-hunting benchmark finds best LLM agent flags only 3.8% of malicious events in raw logs",
 "lane": "defense",
 "kind": "benchmark",
 "summary": "A technical report from security vendor Simbian AI presents the Cyber Defense Benchmark, which asks agents to hunt through 75,000-135,000 raw Windows event log records per episode, with no guiding questions, and flag the timestamps of malicious events drawn from 106 OTRF attack procedures. In the first version, the best of five frontier models (Claude Opus 4.6) flagged only 3.8% of malicious events on average and no model met the authors' bar of 50% recall on every ATT&CK tactic. A revision two days later, with more models and a new coverage metric, reached the same no-pass conclusion.",
 "whyItMatters": "It contrasts strong LLM scores on curated security Q&A with very weak performance on open-ended threat hunting.",
 "actors": [
  "simbian-ai"
 ],
 "topics": [
  "soc-automation",
  "capability-evaluation",
  "eval-validity"
 ],
 "atlas": [
  "eval-environment"
 ],
 "artifacts": [
  "cyber-defense-benchmark",
  "claude-opus-4",
  "gemini",
  "gpt-5-family",
  "kimi"
 ],
 "sources": [
  {
   "url": "https://arxiv.org/abs/2604.19533",
   "publisher": "arXiv",
   "title": "Cyber Defense Benchmark: Agentic Threat Hunting Evaluation for LLMs in SecOps",
   "date": "2026-04-21",
   "type": "primary",
   "accessed": "2026-09-25"
  }
 ],
 "keyFacts": [
  {
   "fact": "106 attack procedures from the OTRF Security-Datasets corpus wrapped in a Gymnasium environment; each episode is a SQLite database of 75,000-135,000 log records; 26 campaigns cover 105 of the 106 procedures.",
   "locator": "arXiv abstract; v1 PDF, seed selection section"
  },
  {
   "fact": "v1 (21 Apr 2026), five models (Claude Opus 4.6, GPT-5, Gemini 3.1 Pro, Kimi K2.5, Gemini 3 Flash): Claude Opus 4.6 submitted correct flags for 3.82% of malicious events on average and cleared the 50%-recall bar on 5 of 13 tactics; the other four models cleared none.",
   "locator": "v1 PDF, Table 4 and passing-score section; arXiv abstract"
  },
  {
   "fact": "v3 (23 Apr 2026) evaluates 11 models and switches the headline metric to a Coverage Score: Claude Opus 4.6 still leads at 0.55 (±0.05) coverage and 4.48% of flags found, clearing 7 of 13 tactics; no model passes.",
   "locator": "v3 PDF, Table 5 and Section 9.2"
  }
 ],
 "significance": 3,
 "fideQuestions": [
  "FID-075",
  "FID-076"
 ],
 "methods": [
  "ctf-benchmarks"
 ],
 "review": "assistant-drafted",
 "addedOn": "2026-09-25"
}