{
 "license": "CC-BY-4.0",
 "attribution": "Fide AI, Agentic Cyber Explorer",
 "url": "https://agentic-cyber-explorer.pages.dev/events/secalign-preference-optimization-2024/",
 "asOf": "2026-09-26",
 "id": "secalign-preference-optimization-2024",
 "date": "2024-10-07",
 "datePrecision": "day",
 "title": "SecAlign uses preference optimization to train LLMs against prompt injection",
 "lane": "defense",
 "kind": "paper",
 "summary": "Chen and colleagues (UC Berkeley and Meta) train models with preference optimization to prefer responses that follow the legitimate instruction over those that follow injected instructions. In the ACM CCS 2025 version they report injection success rates below 10% even for attacks more sophisticated than those seen in training, with utility similar to the undefended model; the October 2024 first version reported GCG-based injection success on Mistral-7B falling from 56% to 2%.",
 "whyItMatters": "SecAlign and its open-weight successor Meta SecAlign are leading training-time defenses whose robustness claims were later tested by adaptive attacks.",
 "actors": [
  "meta",
  "uc-berkeley"
 ],
 "topics": [
  "prompt-injection"
 ],
 "atlas": [
  "model",
  "untrusted-content"
 ],
 "artifacts": [
  "secalign"
 ],
 "sources": [
  {
   "url": "https://arxiv.org/abs/2410.05451",
   "publisher": "arXiv",
   "title": "SecAlign: Defending Against Prompt Injection with Preference Optimization",
   "date": "2024-10-07",
   "type": "primary",
   "accessed": "2026-09-25"
  }
 ],
 "keyFacts": [
  {
   "fact": "CCS 2025 version (arXiv v3, July 2025): prompt injection success rates below 10% across tested attacks, including ones much more sophisticated than those seen in training.",
   "locator": "Abstract (v3)"
  },
  {
   "fact": "First version (October 7, 2024, then titled 'Aligning LLMs to Be Robust Against Prompt Injection'): GCG-based injection success on Mistral-7B fell from 56% to 2%.",
   "locator": "Abstract (v1)"
  }
 ],
 "significance": 3,
 "fideQuestions": [],
 "methods": [
  "indirect-prompt-injection",
  "instruction-priority-training"
 ],
 "review": "assistant-drafted",
 "addedOn": "2026-09-25"
}