{
 "license": "CC-BY-4.0",
 "attribution": "Fide AI, Agentic Cyber Explorer",
 "url": "https://agentic-cyber-explorer.pages.dev/events/alertbench-aida-evidence-based-soc-triage-2026/",
 "asOf": "2026-10-30",
 "id": "alertbench-aida-evidence-based-soc-triage-2026",
 "date": "2026-10-07",
 "datePrecision": "day",
 "title": "ALERT-BENCH finds triage failures; AIDA reduces missed attacks in telemetry replay",
 "lane": "defense",
 "kind": "benchmark",
 "summary": "Researchers introduce ALERT-BENCH, requiring agents to investigate detector-generated alerts through a live SIEM over replayed public telemetry. On the main 1,247-alert sample, five reasoning approaches each miss at least 40.4% of attack-related alerts, while the authors’ AIDA architecture reduces the false-negative rate to 3.1% under the same GPT-4.1 configuration, with more alerts escalated to analysts. Additional scenario and model tests remain within the studied dataset family; analyst use and independent SOC validation are untested.",
 "whyItMatters": "It connects triage decisions to evidence retrieval and independent challenge, showing both improved attack coverage and the resulting analyst workload.",
 "actors": [
  "virginia-tech",
  "north-carolina-state-university",
  "google"
 ],
 "topics": [
  "soc-automation",
  "autonomous-defense",
  "capability-evaluation"
 ],
 "atlas": [
  "tools",
  "other-agents"
 ],
 "artifacts": [
  "gpt-4-family"
 ],
 "sources": [
  {
   "url": "https://arxiv.org/abs/2610.10608v1",
   "publisher": "arXiv",
   "title": "From Investigation Failures to Reliable SOC Agents: Understanding and Improving LLM-Based Alert Triage",
   "date": "2026-10-07",
   "type": "primary",
   "accessed": "2026-10-13"
  }
 ],
 "keyFacts": [
  {
   "fact": "The main comparison uses 1,247 alerts, including 225 attack-related and 1,022 benign alerts, from a four-day AIT-LDSv2 scenario. Each approach uses GPT-4.1 and the same evidence interface; investigators begin from the alert rather than a preselected evidence packet.",
   "locator": "Sections III–IV; Table I"
  },
  {
   "fact": "AIDA records a proposed decision before separate challenge and adjudication. On the main sample its reported F1 is 0.958 versus 0.371–0.744 for the five baselines; false-negative rate falls to 3.1% from the best baseline’s 40.4%, while the share escalated rises to 18.4% from baseline rates of 8.5–14.0%.",
   "locator": "Abstract; Section VI; Figure 6"
  },
  {
   "fact": "The authors also test five open-weight models on a 300-alert subset and evaluate a separate five-day scenario containing 337 alerts with 5.93% attack prevalence. These provide broader tests within the same dataset family, not evaluation on independently collected operational SOC data. On the 300-alert subset Qwen3.5 9B reaches F1 0.952, while the other tested open-weight models range from 0.098 to 0.777, showing substantial backend dependence.",
   "locator": "Section VI-D; Discussion and Limitations"
  },
  {
   "fact": "The evaluation begins after a detector raises an alert and depends on the accessible investigation context. Its experiments do not measure analysts’ review effort, override decisions or effectiveness in operational deployment.",
   "locator": "Discussion and Limitations"
  }
 ],
 "significance": 3,
 "fideQuestions": [
  "FID-075",
  "FID-076"
 ],
 "methods": [
  "ai-monitoring"
 ],
 "review": "assistant-drafted",
 "addedOn": "2026-10-13",
 "revisions": []
}