{
 "license": "CC-BY-4.0",
 "attribution": "Fide AI, Agentic Cyber Explorer",
 "url": "https://agentic-cyber-explorer.pages.dev/events/meta-crowdstrike-cybersoceval-2025/",
 "asOf": "2026-09-26",
 "id": "meta-crowdstrike-cybersoceval-2025",
 "date": "2025-09-24",
 "datePrecision": "day",
 "title": "Meta and CrowdStrike release CyberSOCEval benchmarks for malware analysis and threat intel reasoning",
 "lane": "defense",
 "kind": "benchmark",
 "summary": "CyberSOCEval adds two open-source SOC benchmarks to CyberSecEval 4: malware analysis questions built from sandbox detonation reports, and threat intelligence reasoning over unstructured reports. The authors find larger, newer models do better, reasoning models gain less than in coding and math, and current models are far from saturating the tasks.",
 "whyItMatters": "It gives defenders an open benchmark grounded in real sandbox and threat-report data rather than generic security trivia.",
 "actors": [
  "meta",
  "crowdstrike"
 ],
 "topics": [
  "soc-automation",
  "capability-evaluation",
  "threat-intelligence"
 ],
 "atlas": [
  "eval-environment"
 ],
 "artifacts": [
  "cybersoceval",
  "cyberseceval"
 ],
 "sources": [
  {
   "url": "https://arxiv.org/abs/2509.20166",
   "publisher": "arXiv",
   "title": "CyberSOCEval: Benchmarking LLMs Capabilities for Malware Analysis and Threat Intelligence Reasoning",
   "date": "2025-09-24",
   "type": "primary",
   "accessed": "2026-09-25"
  },
  {
   "url": "https://github.com/CrowdStrike/CyberSOCEval_data",
   "publisher": "GitHub (CrowdStrike)",
   "title": "CyberSOCEval_data",
   "type": "primary",
   "accessed": "2026-09-25"
  }
 ],
 "keyFacts": [
  {
   "fact": "CyberSOCEval covers two tasks, Malware Analysis and Threat Intelligence Reasoning, within CyberSecEval 4.",
   "locator": "Abstract"
  },
  {
   "fact": "Reasoning models using test-time scaling do not get the boost seen in coding and math; models are far from saturating the benchmark.",
   "locator": "Abstract"
  }
 ],
 "significance": 3,
 "fideQuestions": [
  "FID-075",
  "FID-076"
 ],
 "methods": [
  "ctf-benchmarks",
  "sandboxing-egress"
 ],
 "review": "assistant-drafted",
 "addedOn": "2026-09-25"
}