{ "schemaVersion": "4.0", "snapshotId": "2026-10-25.5", "verifiedOn": "2026-10-25", "title": "NUWA Frontier AI Safety Index", "coverage": { "en": "The catalogue contains 137 original research works across five research functions. Source screening, affiliations and protocol details are documented separately from numerical results.", "zh": "目录收录 137 项原始研究，覆盖五类研究功能。来源筛选、机构署名及协议细节与数值结果分别记录。" }, "sources": [ { "id": "agentcyberrange", "title": "AgentCyberRange: Evaluate Agent Cyberattack Capabilities in Enterprise-like Cyber Ranges", "url": "https://agentcyberrange.io/", "version": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "publicationDate": null, "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "https://agentcyberrange.io" }, { "id": "agentdojo", "title": "AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents", "url": "https://arxiv.org/html/2406.13352v3", "version": "arXiv:2406.13352v3", "publicationDate": "2024-11-24", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2406.13352", "authors": [ "Edoardo Debenedetti", "Jie Zhang", "Mislav Balunovic", "Luca Beurer-Kellner   Marc Fischer   Florian Tramèr" ], "aliases": [ "https://doi.org/10.48550/arXiv.2406.13352", "https://arxiv.org/abs/2406.13352", "https://arxiv.org/pdf/2406.13352" ] }, { "id": "cybench", "title": "Cybench: A Framework for Evaluating Cybersecurity Capabilities and Risks of Language Models", "url": "https://arxiv.org/abs/2408.08926v4", "version": "arXiv:2408.08926v4", "publicationDate": "2025-04-12", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2408.08926", "authors": [ "Andy K. Zhang", "Neil Perry", "Riya Dulepet", "Joey Ji", "Celeste Menders", "Justin W. Lin", "Eliot Jones" ], "aliases": [ "https://doi.org/10.48550/arXiv.2408.08926", "https://arxiv.org/abs/2408.08926", "https://arxiv.org/pdf/2408.08926" ] }, { "id": "cybergym", "title": "CyberGym: Evaluating AI Agents' Real-World Cybersecurity Capabilities at Scale", "url": "https://arxiv.org/abs/2506.02548v3", "version": "arXiv:2506.02548v3", "publicationDate": "2026-03-24", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2506.02548", "authors": [ "Zhun Wang", "Tianneng Shi 1 1 footnotemark: 1", "Jingxuan He", "Matthew Cai", "Jialin Zhang", "Dawn Song", "{ zhun.wang", "stneng }@berkeley.edu" ], "aliases": [ "https://doi.org/10.48550/arXiv.2506.02548", "https://arxiv.org/abs/2506.02548", "https://arxiv.org/pdf/2506.02548" ] }, { "id": "agentharm", "title": "AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents", "url": "https://arxiv.org/abs/2410.09024v3", "version": "arXiv:2410.09024v3", "publicationDate": "2025-04-18", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2410.09024", "authors": [ "Maksym Andriushchenko", "Alexandra Souly", "Mateusz Dziemian", "Derek Duenas", "Maxwell Lin", "Justin Wang", "Dan Hendrycks", "Andy Zou", "Zico Kolter", "Matt Fredrikson", "Eric Winsor", "Jerome Wynne", "Yarin Gal", "Xander Davies" ], "aliases": [ "https://doi.org/10.48550/arXiv.2410.09024", "https://arxiv.org/abs/2410.09024", "https://arxiv.org/pdf/2410.09024" ] }, { "id": "harmbench", "title": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/abs/2402.04249v2", "version": "arXiv:2402.04249v2", "publicationDate": "2024-02-27", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2402.04249", "authors": [ "Mantas Mazeika", "Long Phan", "Xuwang Yin", "Andy Zou", "Zifan Wang", "Norman Mu", "Elham Sakhaee", "Nathaniel Li", "Steven Basart", "Bo Li", "David Forsyth", "Dan Hendrycks" ], "aliases": [ "https://doi.org/10.48550/arXiv.2402.04249", "https://arxiv.org/abs/2402.04249", "https://arxiv.org/pdf/2402.04249" ] }, { "id": "strongreject", "title": "A StrongREJECT for Empty Jailbreaks", "url": "https://arxiv.org/abs/2402.10260v2", "version": "arXiv:2402.10260v2", "publicationDate": "2024-08-27", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2402.10260", "authors": [ "Alexandra Souly ∗", "Qingyuan Lu ∗", "Dillon Bowen ∗", "Tu Trinh †", "Elvis Hsieh †", "Sana Pandey", "Pieter Abbeel", "Justin Svegliato", "Scott Emmons ‡", "Olivia Watkins ‡", "Sam Toyer ‡", "Center for Human-Compatible AI", "UC Berkeley" ], "aliases": [ "https://doi.org/10.48550/arXiv.2402.10260", "https://arxiv.org/abs/2402.10260", "https://arxiv.org/pdf/2402.10260" ] }, { "id": "replibench", "title": "RepliBench: Evaluating the Autonomous Replication Capabilities of Language Model Agents", "url": "https://arxiv.org/abs/2504.18565v2", "version": "arXiv:2504.18565v2", "publicationDate": "2025-05-05", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2504.18565", "authors": [ "Sid Black *", "Asa Cooper Stickland *", "Jake Pencharz", "Oliver Sourbut" ], "aliases": [ "https://doi.org/10.48550/arXiv.2504.18565", "https://arxiv.org/abs/2504.18565", "https://arxiv.org/pdf/2504.18565" ] }, { "id": "self-replication", "title": "Frontier AI systems have surpassed the self-replicating red line", "url": "https://arxiv.org/html/2412.12140v1", "version": "arXiv:2412.12140v1", "publicationDate": "2024-12-09", "datePrecision": "day", "type": "controlled-study", "verifiedOn": "2026-10-24", "documentId": "arxiv:2412.12140", "authors": [ "Xudong Pan (潘旭东)", "Jiarun Dai (戴嘉润)", "Yihe Fan (范一禾)", "Min Yang (杨珉)" ], "aliases": [ "https://doi.org/10.48550/arXiv.2412.12140", "https://arxiv.org/abs/2412.12140", "https://arxiv.org/pdf/2412.12140" ] }, { "id": "opendeception", "title": "OpenDeception: Learning Deception and Trust in Human-AI Interaction via Multi-Agent Simulation", "url": "https://arxiv.org/abs/2504.13707v3", "version": "arXiv:2504.13707v3", "publicationDate": "2026-02-06", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2504.13707", "authors": [ "Yichen Wu", "Qianqian Gao", "Xudong Pan", "Geng Hong", "Min Yang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2504.13707", "https://arxiv.org/abs/2504.13707", "https://arxiv.org/pdf/2504.13707" ] }, { "id": "in-context-scheming", "title": "Frontier Models are Capable of In-context Scheming", "url": "https://arxiv.org/abs/2412.04984v2", "version": "arXiv:2412.04984v2", "publicationDate": "2025-01-14", "datePrecision": "day", "type": "controlled-study", "verifiedOn": "2026-10-24", "documentId": "arxiv:2412.04984", "authors": [ "Alexander Meinke", "Bronson Schoen 1 1 footnotemark: 1", "Jérémy Scheurer 1 1 footnotemark: 1", "Mikita Balesni", "Rusheb Shah", "Marius Hobbhahn", "Apollo Research" ], "aliases": [ "https://doi.org/10.48550/arXiv.2412.04984", "https://arxiv.org/abs/2412.04984", "https://arxiv.org/pdf/2412.04984" ] }, { "id": "stealth-evals", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/abs/2505.01420v4", "version": "arXiv:2505.01420v4", "publicationDate": "2025-07-03", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2505.01420", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "aliases": [ "https://doi.org/10.48550/arXiv.2505.01420", "https://arxiv.org/abs/2505.01420", "https://arxiv.org/pdf/2505.01420" ] }, { "id": "re-bench", "title": "RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts", "url": "https://arxiv.org/abs/2411.15114v2", "version": "arXiv:2411.15114v2", "publicationDate": "2025-05-27", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2411.15114", "authors": [ "Hjalmar Wijk", "Tao Lin", "Joel Becker", "Sami Jawhar", "Neev Parikh", "Thomas Broadley", "Lawrence Chan 2 2 footnotemark: 2", "Michael Chen 2 2 footnotemark: 2", "Josh Clymer 2 2 footnotemark: 2", "Jai Dhyani 2 2 footnotemark: 2", "Elena Ericheva 2 2 footnotemark: 2", "Katharyn Garcia 2 2 footnotemark: 2", "Brian Goodrich 2 2 footnotemark: 2", "Nikola Jurkovic 2 2 footnotemark: 2", "Holden Karnofsky 2 2 footnotemark: 2", "Megan Kinniment 2 2 footnotemark: 2", "Aron Lajko 2 2 footnotemark: 2 3 3 footnotemark: 3", "Seraphina Nix 2 2 footnotemark: 2", "Lucas Sato 2 2 footnotemark: 2", "William Saunders 2 2 footnotemark: 2", "Maksym Taran 2 2 footnotemark: 2", "Ben West 2 2 footnotemark: 2", "Elizabeth Barnes", "Model Evaluation and Threat Research (METR)" ], "aliases": [ "https://doi.org/10.48550/arXiv.2411.15114", "https://arxiv.org/abs/2411.15114", "https://arxiv.org/pdf/2411.15114" ] }, { "id": "time-horizons", "title": "Task-Completion Time Horizons of Frontier AI Models", "url": "https://metr.org/time-horizons/", "version": "Time Horizon 1.1; page updated 2026-09-08", "publicationDate": "2026-09-08", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "https://metr.org/time-horizons", "authors": [ "Thomas Kwa", "Ben West 1 1 footnotemark: 1", "Joel Becker", "Amy Deng", "Katharyn GarciaMax Hasin , Sami Jawhar , Megan Kinniment , Nate Rush , Sydney Von ArxRyan Bloom , Thomas Broadley , Haoxing Du , Brian Goodrich , Nikola Jurkovic , Luke Harold Miles , Seraphina Nix , Tao Lin , Chris Painter , Neev Parikh , David Rein , Lucas Jun Koba Sato , Hjalmar Wijk , Daniel M. ZieglerElizabeth Barnes , Lawrence Chan Model Evaluation & Threat Research (METR)" ] }, { "id": "wmdp", "title": "The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "url": "https://arxiv.org/abs/2403.03218v7", "version": "arXiv:2403.03218v7", "publicationDate": "2024-05-15", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2403.03218", "authors": [ "Nathaniel Li ∗", "Alexander Pan ∗", "Anjali Gopal †", "Summer Yue †", "Daniel Berrios †", "Alice Gatti ‡", "Justin D. Li ‡", "Ann-Kathrin Dombrowski ‡", "Shashwat Goel ‡", "Long Phan ‡", "Gabriel Mukobi", "Nathan Helm-Burger", "Rassin Lababidi", "Lennart Justen", "Andrew B. Liu", "Michael Chen", "Isabelle Barrass", "Oliver Zhang", "Xiaoyuan Zhu", "Rishub Tamirisa", "Bhrugu Bharathi", "Adam Khoja", "Zhenqi Zhao", "Ariel Herbert-Voss", "Cort B. Breuer", "Samuel Marks", "Oam Patel", "Andy Zou", "Mantas Mazeika", "Zifan Wang", "Palash Oswal", "Weiran Lin", "Adam A. Hunt", "Justin Tienken-Harder", "Kevin Y. Shih", "Kemper Talley", "John Guan", "Russell Kaplan", "Ian Steneker", "David Campbell", "Brad Jokubaitis", "Alex Levinson", "Jean Wang", "William Qian", "Kallol Krishna Karmakar", "Steven Basart", "Stephen Fitz", "Mindy Levine", "Ponnurangam Kumaraguru", "Uday Tupakula", "Vijay Varadharajan", "Ruoyu Wang", "Yan Shoshitaishvili", "Jimmy Ba", "Kevin M. Esvelt", "Alexandr Wang ∗∗", "Dan Hendrycks ∗∗" ], "aliases": [ "https://doi.org/10.48550/arXiv.2403.03218", "https://arxiv.org/abs/2403.03218", "https://arxiv.org/pdf/2403.03218" ] }, { "id": "political-persuasion", "title": "Benchmarking Political Persuasion Risks Across Frontier Large Language Models", "url": "https://arxiv.org/abs/2603.09884v1", "version": "arXiv:2603.09884v1", "publicationDate": "2026-03-10", "datePrecision": "day", "type": "controlled-study", "verifiedOn": "2026-10-24", "documentId": "arxiv:2603.09884", "authors": [ "Zhongren Chen", "Joshua Kalla", "Quan Le" ], "aliases": [ "https://doi.org/10.48550/arXiv.2603.09884", "https://arxiv.org/abs/2603.09884", "https://arxiv.org/pdf/2603.09884" ] }, { "id": "asimov", "title": "Can AI Perceive Physical Danger and Intervene?", "url": "https://asimov-benchmark.github.io/v2/", "version": "ASIMOV v2 / arXiv:2509.21651", "publicationDate": null, "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "https://asimov-benchmark.github.io/v2", "authors": [ "Abhishek Jindal", "Dmitry Kalashnikov", "R. Alex Hofer", "Oscar Chang", "Divya Garikapati", "Anirudha Majumdar", "Pierre Sermanet", "Vikas Sindhwani" ] }, { "id": "anthropic-cards", "title": "Anthropic model system cards", "url": "https://www.anthropic.com/system-cards", "version": "Index checked 2026-10-25", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "official-model-report", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/system-cards" }, { "id": "anthropic-unintended", "title": "Investigating unintended model actions in our evaluations and internal use", "url": "https://www.anthropic.com/research/investigating-unintended-model-actions", "version": "2026-10-09 article", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "research-report", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/research/investigating-unintended-model-actions" }, { "id": "openai-cards", "title": "OpenAI Deployment Safety Hub", "url": "https://deploymentsafety.openai.com/", "version": "Index checked 2026-10-25", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "official-model-report", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com" }, { "id": "openai-scheming", "title": "Detecting and reducing scheming in AI models", "url": "https://openai.com/index/detecting-and-reducing-scheming-in-ai-models/", "version": "2025-09-17 article", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "research-report", "verifiedOn": "2026-10-25", "documentId": "https://openai.com/index/detecting-and-reducing-scheming-in-ai-models" }, { "id": "gemini-cards", "title": "Google DeepMind model cards", "url": "https://deepmind.google/models/model-cards/", "version": "Index checked 2026-10-25", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "official-model-report", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards" }, { "id": "gemini-dangerous", "title": "Evaluating Frontier Models for Dangerous Capabilities", "url": "https://deepmind.google/research/publications/78150/", "version": "2024-03-21 publication", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "research-report", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/research/publications/78150" }, { "id": "muse-spark", "title": "Muse Spark Safety & Preparedness Report", "url": "https://arxiv.org/abs/2606.12429", "version": "arXiv:2606.12429", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "official-model-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2606.12429", "aliases": [ "https://doi.org/10.48550/arXiv.2606.12429", "https://arxiv.org/abs/2606.12429", "https://arxiv.org/pdf/2606.12429" ] }, { "id": "apollo-science", "title": "Apollo Research: Science of Scheming", "url": "https://www.apolloresearch.ai/science", "version": "Index checked 2026-10-25", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "research-index", "verifiedOn": "2026-10-25", "documentId": "https://www.apolloresearch.ai/science" }, { "id": "aisi-research", "title": "UK AI Security Institute research", "url": "https://www.aisi.gov.uk/research", "version": "Index checked 2026-10-25", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "research-index", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research" }, { "id": "aisi-propensity", "title": "Propensity Inference: Environmental Contributors to LLM Behaviour", "url": "https://www.aisi.gov.uk/research/propensity-inference-environmental-contributors-to-llm-behaviour", "version": "Research page checked 2026-10-25", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "research-report", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/propensity-inference-environmental-contributors-to-llm-behaviour", "authors": [ "Olli Järviniemi", "Oliver Makins", "Jacob Merizian", "Robert Kirk", "Ben Millwood" ] }, { "id": "aisi-cyber-range", "title": "Measuring AI Agents’ Progress on Multi-Step Cyber Attack Scenarios", "url": "https://www.aisi.gov.uk/research/measuring-ai-agents-progress-on-multi-step-cyber-attack-scenarios", "version": "Research page checked 2026-10-25", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "research-report", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/measuring-ai-agents-progress-on-multi-step-cyber-attack-scenarios", "authors": [ "Linus Folkerts", "Will Payne", "Simon Inman", "Philippos Giavridis", "Joe Skinner", "Sam Deverett †", "James Aung", "Ekin Zorer", "Michael Schmatz", "Mahmoud Ghanem", "John Wilkinson", "Alan Steer", "Vy Hong", "Jessica Wang" ] }, { "id": "far-auditing", "title": "Auditing Games for Sandbagging", "url": "https://www.far.ai/research/auditing-games-for-sandbagging", "version": "2025-12-07 article / arXiv:2512.07810", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "research-report", "verifiedOn": "2026-10-25", "documentId": "https://www.far.ai/research/auditing-games-for-sandbagging" }, { "id": "palisade-shutdown", "title": "Shutdown resistance in reasoning models", "url": "https://palisaderesearch.org/research/shutdown-resistance", "version": "2025-07-05 article; later TMLR version separate", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "research-report", "verifiedOn": "2026-10-25", "documentId": "https://palisaderesearch.org/research/shutdown-resistance", "authors": [ "Jeremy Schlatter", "Benjamin Weinstein-Raun", "Jeffrey Ladish" ] }, { "id": "truthful-sad", "title": "Me, Myself, and AI: The Situational Awareness Dataset for LLMs", "url": "https://truthful.ai/papers/situational-awareness-dataset/", "version": "2024-07-15 article; paper linked", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "https://truthful.ai/papers/situational-awareness-dataset", "authors": [ "Rudolf Laine", "Bilal Chughtai", "Jan Betley", "Kaivalya Hariharan", "Jérémy Scheurer", "Mikita Balesni", "Marius Hobbhahn", "Alexander Meinke", "Owain Evans" ] }, { "id": "palisade-robot-shutdown", "title": "Shutdown Resistance in Large Language Models, on robots!", "url": "https://palisaderesearch.org/research/shutdown-resistance-on-robots", "version": "2026-02-12 technical report", "publicationDate": "2026-02-12", "datePrecision": "day", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "https://palisaderesearch.org/research/shutdown-resistance-on-robots", "authors": [ "Jeremy Schlatter", "Benjamin Weinstein-Raun", "Jeffrey Ladish" ] }, { "id": "secbench-pro", "title": "SEC-bench Pro: Can Language Models Solve Long-Horizon Software Security Tasks?", "url": "https://arxiv.org/abs/2605.26548v2", "version": "arXiv:2605.26548v2", "publicationDate": "2026-07-20", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "arxiv:2605.26548", "authors": [ "Hwiwon Lee ∗ Jiawei Liu \nDongjun Kim \nWubing Xia \nZiqi Zhang \nChunqiu Steven Xia \nLingming Zhang ∗" ], "aliases": [ "https://doi.org/10.48550/arXiv.2605.26548", "https://arxiv.org/abs/2605.26548", "https://arxiv.org/pdf/2605.26548" ] }, { "id": "sandboxescapebench", "title": "Quantifying Frontier LLM Capabilities for Container Sandbox Escape", "url": "https://www.aisi.gov.uk/research/quantifying-frontier-llm-capabilities-for-container-sandbox-escape", "version": "AISI research page checked 2026-10-25", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/quantifying-frontier-llm-capabilities-for-container-sandbox-escape", "authors": [ "Rahul Marchand", "Art O Cathain", "Jerome Wynne", "Philippos Maximos Giavridis", "Stuart Jennings", "Freddy Tuxworth", "Tolga H. Dur", "Sam Deverett", "John Wilkinson", "Jason Gwartz ∗", "Harry Coppock ∗" ] }, { "id": "paperbench", "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research", "url": "https://arxiv.org/abs/2504.01848v3", "version": "arXiv:2504.01848v3", "publicationDate": "2025-04-07", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "arxiv:2504.01848", "authors": [ "Starace, Giulio", "Jaffe, Oliver", "Sherburn, Dane", "Aung, James", "Chan, Jun Shern", "Maksin, Leon", "Dias, Rachel", "Mays, Evan", "Kinsella, Benjamin", "Thompson, Wyatt", "Heidecke, Johannes", "Glaese, Amelia", "Patwardhan, Tejal" ], "aliases": [ "https://doi.org/10.48550/arXiv.2504.01848", "https://arxiv.org/abs/2504.01848", "https://arxiv.org/pdf/2504.01848" ] }, { "id": "alignment-faking", "title": "Alignment faking in large language models", "url": "https://www.anthropic.com/research/alignment-faking", "version": "Anthropic research article 2024-12-18", "publicationDate": "2024-12-18", "datePrecision": "day", "type": "controlled-study", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/research/alignment-faking", "authors": [ "Ryan Greenblatt", "† Carson Denison † † footnotemark:", "Benjamin Wright † † footnotemark:", "Fabien Roger † † footnotemark:", "Monte MacDiarmid † † footnotemark:" ] }, { "id": "agentic-misalignment", "title": "Agentic misalignment: How LLMs could be insider threats", "url": "https://www.anthropic.com/research/agentic-misalignment", "version": "Anthropic research article 2025-06-20", "publicationDate": "2025-06-20", "datePrecision": "day", "type": "controlled-study", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/research/agentic-misalignment" }, { "id": "shade-arena", "title": "SHADE-Arena: Evaluating sabotage and monitoring in LLM agents", "url": "https://www.anthropic.com/research/shade-arena-sabotage-monitoring", "version": "Anthropic research article 2025-06-16", "publicationDate": "2025-06-16", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/research/shade-arena-sabotage-monitoring" }, { "id": "vct", "title": "Virology Capabilities Test (VCT): A Multimodal Virology Q&A Benchmark", "url": "https://arxiv.org/abs/2504.16137v2", "version": "arXiv:2504.16137v2", "publicationDate": "2025-04-29", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "arxiv:2504.16137", "authors": [ "Jasper Götting a", "Pedro Medeiros a", "Jon G Sanders a", "Nathaniel Li †", "Long Phan", "Karam Elabd", "Lennart Justen", "Dan Hendrycks", "Seth Donoughe ∗" ], "aliases": [ "https://doi.org/10.48550/arXiv.2504.16137", "https://arxiv.org/abs/2504.16137", "https://arxiv.org/pdf/2504.16137" ] }, { "id": "vct-v2", "title": "Introducing VCT-v2 — the updated Virology Capabilities Test", "url": "https://securebio.org/blog/introducing-vct-v2/", "version": "SecureBio release 2026-09-11", "publicationDate": "2026-09-11", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "https://securebio.org/blog/introducing-vct-v2" }, { "id": "biotier", "title": "BioTIER: A Refusal Benchmark for Targeted Biological Risk Mitigation", "url": "https://arxiv.org/abs/2607.14479v1", "version": "arXiv:2607.14479v1", "publicationDate": "2026-07-16", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "arxiv:2607.14479", "authors": [ "Eleanor M. Marshall", "Pedro Medeiros", "Peter Peneder", "Nelly Mak", "Jacob Kaffey", "Faith Rovenolt", "Mac Walker", "Seth Donoughe", "Jasper Götting" ], "aliases": [ "https://doi.org/10.48550/arXiv.2607.14479", "https://arxiv.org/abs/2607.14479", "https://arxiv.org/pdf/2607.14479" ] }, { "id": "mask", "title": "The MASK Benchmark: Disentangling Honesty From Accuracy in AI Systems", "url": "https://arxiv.org/abs/2503.03750v3", "version": "arXiv:2503.03750v3", "publicationDate": "2026-01-05", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "arxiv:2503.03750", "authors": [ "Richard Ren", "Arunim Agarwal", "Mantas Mazeika", "Cristina Menghini", "Robert Vacareanu", "Brad Kenstler", "Mick Yang", "Isabelle Barrass", "Alice Gatti", "Xuwang Yin", "Eduardo Trevino", "Matias Geralnik", "Adam Khoja", "Dean Lee", "Summer Yue", "Dan Hendrycks" ], "aliases": [ "https://doi.org/10.48550/arXiv.2503.03750", "https://arxiv.org/abs/2503.03750", "https://arxiv.org/pdf/2503.03750" ] }, { "id": "ape", "title": "It's the Thought that Counts: Evaluating the Attempts of Frontier LLMs to Persuade on Harmful Topics", "url": "https://arxiv.org/abs/2506.02873v4", "version": "arXiv:2506.02873v4", "publicationDate": "2026-02-15", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "arxiv:2506.02873", "authors": [ "\\authorname Matthew Kowal \\authorinstitution FAR.AI \\authorname Jasper Timm \\authorinstitution FAR.AI \\authorname Jean-Francois Godbout \\authorinstitution Université de Montréal, MILA \\authorname Thomas Costello \\authorinstitution Carnegie Mellon University \\authorname Antonio A. Arechar \\authorinstitution MIT, Center for Research and Teaching in Economics \\authorname Gordon Pennycook \\authorinstitution Cornell University, University of Regina \\authorname David Rand \\authorinstitution Cornell University, MIT \\authorname Adam Gleave \\authorinstitution FAR.AI \\authorname Kellin Pelrine \\authorinstitution FAR.AI" ], "aliases": [ "https://doi.org/10.48550/arXiv.2506.02873", "https://arxiv.org/abs/2506.02873", "https://arxiv.org/pdf/2506.02873" ] }, { "id": "conspiracy-persuasion", "title": "Large language models can effectively convince people to believe conspiracies", "url": "https://www.far.ai/research/large-language-models-can-effectively-convince-people-to-believe-conspiracies", "version": "FAR.AI report 2026-01-08", "publicationDate": "2026-01-08", "datePrecision": "day", "type": "controlled-study", "verifiedOn": "2026-10-25", "documentId": "https://www.far.ai/research/large-language-models-can-effectively-convince-people-to-believe-conspiracies" }, { "id": "agentcyberrange-research-introduction", "title": "AgentCyberRange: Evaluating autonomous cyber capabilities over long-horizon tasks", "url": "https://whitzard.tech/nuwa/blog/agentcyberrange/", "version": "Source capture 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://whitzard.tech/nuwa/blog/agentcyberrange" }, { "id": "asimov-author-source", "title": "Can AI Perceive Physical Danger and Intervene? Thanks: {abjindal, dklakashnikov, oscarchang, divyagari, animajumdar, sindhwani}@google.com", "url": "https://arxiv.org/html/2509.21651", "version": "arXiv:2509.21651v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2509.21651", "authors": [ "Abhishek Jindal", "Dmitry Kalashnikov", "R. Alex Hofer", "Oscar Chang", "Divya Garikapati", "Anirudha Majumdar", "Pierre Sermanet", "Vikas Sindhwani" ], "aliases": [ "https://doi.org/10.48550/arXiv.2509.21651", "https://arxiv.org/abs/2509.21651", "https://arxiv.org/pdf/2509.21651" ] }, { "id": "cybench-author-source", "title": "Cybench: A Framework for Evaluating Cybersecurity Capabilities and Risks of Language Models", "url": "https://arxiv.org/html/2408.08926v4", "version": "arXiv:2408.08926v4", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2408.08926", "authors": [ "Andy K. Zhang", "Neil Perry", "Riya Dulepet", "Joey Ji", "Celeste Menders", "Justin W. Lin", "Eliot Jones" ], "aliases": [ "https://doi.org/10.48550/arXiv.2408.08926", "https://arxiv.org/abs/2408.08926", "https://arxiv.org/pdf/2408.08926" ] }, { "id": "cybergym-author-source", "title": "CyberGym: Evaluating AI Agents’ Real-World Cybersecurity Capabilities at Scale", "url": "https://arxiv.org/html/2506.02548v3", "version": "arXiv:2506.02548v3", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2506.02548", "authors": [ "Zhun Wang", "Tianneng Shi 1 1 footnotemark: 1", "Jingxuan He", "Matthew Cai", "Jialin Zhang", "Dawn Song", "{ zhun.wang", "stneng }@berkeley.edu" ], "aliases": [ "https://doi.org/10.48550/arXiv.2506.02548", "https://arxiv.org/abs/2506.02548", "https://arxiv.org/pdf/2506.02548" ] }, { "id": "in-context-scheming-author-source", "title": "Frontier Models are Capable of In-context Scheming", "url": "https://arxiv.org/html/2412.04984v2", "version": "arXiv:2412.04984v2", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2412.04984", "authors": [ "Alexander Meinke", "Bronson Schoen 1 1 footnotemark: 1", "Jérémy Scheurer 1 1 footnotemark: 1", "Mikita Balesni", "Rusheb Shah", "Marius Hobbhahn", "Apollo Research" ], "aliases": [ "https://doi.org/10.48550/arXiv.2412.04984", "https://arxiv.org/abs/2412.04984", "https://arxiv.org/pdf/2412.04984" ] }, { "id": "aisi-cyber-range-author-source", "title": "Measuring AI Agents’ Progress on Multi-Step Cyber Attack Scenarios", "url": "https://arxiv.org/html/2603.11214", "version": "arXiv:2603.11214v3 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2603.11214", "authors": [ "Linus Folkerts", "Will Payne", "Simon Inman", "Philippos Giavridis", "Joe Skinner", "Sam Deverett †", "James Aung", "Ekin Zorer", "Michael Schmatz", "Mahmoud Ghanem", "John Wilkinson", "Alan Steer", "Vy Hong", "Jessica Wang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2603.11214", "https://arxiv.org/abs/2603.11214", "https://arxiv.org/pdf/2603.11214" ] }, { "id": "time-horizons-author-source", "title": "Measuring AI Ability to Complete Long Software Tasks", "url": "https://arxiv.org/html/2503.14499", "version": "arXiv:2503.14499v4 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2503.14499", "authors": [ "Thomas Kwa", "Ben West 1 1 footnotemark: 1", "Joel Becker", "Amy Deng", "Katharyn GarciaMax Hasin , Sami Jawhar , Megan Kinniment , Nate Rush , Sydney Von ArxRyan Bloom , Thomas Broadley , Haoxing Du , Brian Goodrich , Nikola Jurkovic , Luke Harold Miles , Seraphina Nix , Tao Lin , Chris Painter , Neev Parikh , David Rein , Lucas Jun Koba Sato , Hjalmar Wijk , Daniel M. ZieglerElizabeth Barnes , Lawrence Chan Model Evaluation & Threat Research (METR)" ], "aliases": [ "https://doi.org/10.48550/arXiv.2503.14499", "https://arxiv.org/abs/2503.14499", "https://arxiv.org/pdf/2503.14499" ] }, { "id": "re-bench-author-source", "title": "RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts", "url": "https://arxiv.org/html/2411.15114v2", "version": "arXiv:2411.15114v2", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2411.15114", "authors": [ "Hjalmar Wijk", "Tao Lin", "Joel Becker", "Sami Jawhar", "Neev Parikh", "Thomas Broadley", "Lawrence Chan 2 2 footnotemark: 2", "Michael Chen 2 2 footnotemark: 2", "Josh Clymer 2 2 footnotemark: 2", "Jai Dhyani 2 2 footnotemark: 2", "Elena Ericheva 2 2 footnotemark: 2", "Katharyn Garcia 2 2 footnotemark: 2", "Brian Goodrich 2 2 footnotemark: 2", "Nikola Jurkovic 2 2 footnotemark: 2", "Holden Karnofsky 2 2 footnotemark: 2", "Megan Kinniment 2 2 footnotemark: 2", "Aron Lajko 2 2 footnotemark: 2 3 3 footnotemark: 3", "Seraphina Nix 2 2 footnotemark: 2", "Lucas Sato 2 2 footnotemark: 2", "William Saunders 2 2 footnotemark: 2", "Maksym Taran 2 2 footnotemark: 2", "Ben West 2 2 footnotemark: 2", "Elizabeth Barnes", "Model Evaluation and Threat Research (METR)" ], "aliases": [ "https://doi.org/10.48550/arXiv.2411.15114", "https://arxiv.org/abs/2411.15114", "https://arxiv.org/pdf/2411.15114" ] }, { "id": "replibench-author-source", "title": "RepliBench: Evaluating the Autonomous Replication Capabilities of Language Model Agents", "url": "https://arxiv.org/html/2504.18565v2", "version": "arXiv:2504.18565v2", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2504.18565", "authors": [ "Sid Black *", "Asa Cooper Stickland *", "Jake Pencharz", "Oliver Sourbut" ], "aliases": [ "https://doi.org/10.48550/arXiv.2504.18565", "https://arxiv.org/abs/2504.18565", "https://arxiv.org/pdf/2504.18565" ] }, { "id": "truthful-sad-author-source", "title": "Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "url": "https://arxiv.org/html/2407.04694", "version": "arXiv:2407.04694v1 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2407.04694", "authors": [ "Rudolf Laine", "Bilal Chughtai", "Jan Betley", "Kaivalya Hariharan", "Jérémy Scheurer", "Mikita Balesni", "Marius Hobbhahn", "Alexander Meinke", "Owain Evans" ], "aliases": [ "https://doi.org/10.48550/arXiv.2407.04694", "https://arxiv.org/abs/2407.04694", "https://arxiv.org/pdf/2407.04694" ] }, { "id": "stealth-evals-author-source", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "url": "https://arxiv.org/html/2505.01420v4", "version": "arXiv:2505.01420v4", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2505.01420", "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "aliases": [ "https://doi.org/10.48550/arXiv.2505.01420", "https://arxiv.org/abs/2505.01420", "https://arxiv.org/pdf/2505.01420" ] }, { "id": "wmdp-author-source", "title": "The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "url": "https://arxiv.org/html/2403.03218v7", "version": "arXiv:2403.03218v7", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2403.03218", "authors": [ "Nathaniel Li ∗", "Alexander Pan ∗", "Anjali Gopal †", "Summer Yue †", "Daniel Berrios †", "Alice Gatti ‡", "Justin D. Li ‡", "Ann-Kathrin Dombrowski ‡", "Shashwat Goel ‡", "Long Phan ‡", "Gabriel Mukobi", "Nathan Helm-Burger", "Rassin Lababidi", "Lennart Justen", "Andrew B. Liu", "Michael Chen", "Isabelle Barrass", "Oliver Zhang", "Xiaoyuan Zhu", "Rishub Tamirisa", "Bhrugu Bharathi", "Adam Khoja", "Zhenqi Zhao", "Ariel Herbert-Voss", "Cort B. Breuer", "Samuel Marks", "Oam Patel", "Andy Zou", "Mantas Mazeika", "Zifan Wang", "Palash Oswal", "Weiran Lin", "Adam A. Hunt", "Justin Tienken-Harder", "Kevin Y. Shih", "Kemper Talley", "John Guan", "Russell Kaplan", "Ian Steneker", "David Campbell", "Brad Jokubaitis", "Alex Levinson", "Jean Wang", "William Qian", "Kallol Krishna Karmakar", "Steven Basart", "Stephen Fitz", "Mindy Levine", "Ponnurangam Kumaraguru", "Uday Tupakula", "Vijay Varadharajan", "Ruoyu Wang", "Yan Shoshitaishvili", "Jimmy Ba", "Kevin M. Esvelt", "Alexandr Wang ∗∗", "Dan Hendrycks ∗∗" ], "aliases": [ "https://doi.org/10.48550/arXiv.2403.03218", "https://arxiv.org/abs/2403.03218", "https://arxiv.org/pdf/2403.03218" ] }, { "id": "opendeception-author-source", "title": "OpenDeception : Learning Deception and Trust in Human–AI Interaction via Multi-Agent Simulation", "url": "https://arxiv.org/html/2504.13707v3", "version": "arXiv:2504.13707v3", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2504.13707", "authors": [ "Yichen Wu", "Qianqian Gao", "Xudong Pan", "Geng Hong", "Min Yang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2504.13707", "https://arxiv.org/abs/2504.13707", "https://arxiv.org/pdf/2504.13707" ] }, { "id": "aisi-propensity-author-source", "title": "Propensity Inference: Environmental Contributors to Unsanctioned LLM Behaviour", "url": "https://arxiv.org/html/2604.21098", "version": "arXiv:2604.21098v1 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2604.21098", "authors": [ "Olli Järviniemi", "Oliver Makins", "Jacob Merizian", "Robert Kirk", "Ben Millwood" ], "aliases": [ "https://doi.org/10.48550/arXiv.2604.21098", "https://arxiv.org/abs/2604.21098", "https://arxiv.org/pdf/2604.21098" ] }, { "id": "palisade-shutdown-author-source", "title": "Incomplete Tasks Induce Shutdown Resistance in Some Frontier LLMs", "url": "https://arxiv.org/html/2509.14260", "version": "arXiv:2509.14260v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2509.14260", "authors": [ "Jeremy Schlatter", "Benjamin Weinstein-Raun", "Jeffrey Ladish" ], "aliases": [ "https://doi.org/10.48550/arXiv.2509.14260", "https://arxiv.org/abs/2509.14260", "https://arxiv.org/pdf/2509.14260" ] }, { "id": "agentharm-author-source", "title": "AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents", "url": "https://arxiv.org/html/2410.09024v3", "version": "arXiv:2410.09024v3", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2410.09024", "authors": [ "Maksym Andriushchenko", "Alexandra Souly", "Mateusz Dziemian", "Derek Duenas", "Maxwell Lin", "Justin Wang", "Dan Hendrycks", "Andy Zou", "Zico Kolter", "Matt Fredrikson", "Eric Winsor", "Jerome Wynne", "Yarin Gal", "Xander Davies" ], "aliases": [ "https://doi.org/10.48550/arXiv.2410.09024", "https://arxiv.org/abs/2410.09024", "https://arxiv.org/pdf/2410.09024" ] }, { "id": "harmbench-author-source", "title": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/html/2402.04249v2", "version": "arXiv:2402.04249v2", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2402.04249", "authors": [ "Mantas Mazeika", "Long Phan", "Xuwang Yin", "Andy Zou", "Zifan Wang", "Norman Mu", "Elham Sakhaee", "Nathaniel Li", "Steven Basart", "Bo Li", "David Forsyth", "Dan Hendrycks" ], "aliases": [ "https://doi.org/10.48550/arXiv.2402.04249", "https://arxiv.org/abs/2402.04249", "https://arxiv.org/pdf/2402.04249" ] }, { "id": "political-persuasion-author-source", "title": "Benchmarking Political Persuasion Risks Across Frontier Large Language Models Thanks: Author names are listed in alphabetical order. This research was deemed exempt by the Yale University Human Subjects Committee. We thank Max Nadeau, Lisa Argyle, Dave Rand, and Ethan Busby for helpful comments. This research was funded by a grant from Coefficient Giving. All remaining errors are our own.", "url": "https://arxiv.org/html/2603.09884v1", "version": "arXiv:2603.09884v1", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2603.09884", "authors": [ "Zhongren Chen", "Joshua Kalla", "Quan Le" ], "aliases": [ "https://doi.org/10.48550/arXiv.2603.09884", "https://arxiv.org/abs/2603.09884", "https://arxiv.org/pdf/2603.09884" ] }, { "id": "palisade-robot-shutdown-author-source", "title": "Incomplete Tasks Induce Shutdown Resistance in Some Frontier LLMs", "url": "https://arxiv.org/html/2509.14260", "version": "arXiv:2509.14260v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2509.14260", "authors": [ "Jeremy Schlatter", "Benjamin Weinstein-Raun", "Jeffrey Ladish" ], "aliases": [ "https://doi.org/10.48550/arXiv.2509.14260", "https://arxiv.org/abs/2509.14260", "https://arxiv.org/pdf/2509.14260" ] }, { "id": "strongreject-author-source", "title": "A StrongREJECT ¯ \\overline{\\hbox{{StrongREJECT}}} for Empty Jailbreaks", "url": "https://arxiv.org/html/2402.10260v2", "version": "arXiv:2402.10260v2", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2402.10260", "authors": [ "Alexandra Souly ∗", "Qingyuan Lu ∗", "Dillon Bowen ∗", "Tu Trinh †", "Elvis Hsieh †", "Sana Pandey", "Pieter Abbeel", "Justin Svegliato", "Scott Emmons ‡", "Olivia Watkins ‡", "Sam Toyer ‡", "Center for Human-Compatible AI", "UC Berkeley" ], "aliases": [ "https://doi.org/10.48550/arXiv.2402.10260", "https://arxiv.org/abs/2402.10260", "https://arxiv.org/pdf/2402.10260" ] }, { "id": "secbench-pro-author-source", "title": "SEC-bench Pro: Can Language Models Solve Long-Horizon Software Security Tasks? Thanks: ∗ Correspondence to: {hwiwonl2, lingming}@illinois.edu .", "url": "https://arxiv.org/html/2605.26548v2", "version": "arXiv:2605.26548v2", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2605.26548", "authors": [ "Hwiwon Lee ∗ Jiawei Liu \nDongjun Kim \nWubing Xia \nZiqi Zhang \nChunqiu Steven Xia \nLingming Zhang ∗" ], "aliases": [ "https://doi.org/10.48550/arXiv.2605.26548", "https://arxiv.org/abs/2605.26548", "https://arxiv.org/pdf/2605.26548" ] }, { "id": "sandboxescapebench-author-source", "title": "Quantifying Frontier LLM Capabilities for Container Sandbox Escape", "url": "https://arxiv.org/html/2603.02277", "version": "arXiv:2603.02277v3 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2603.02277", "authors": [ "Rahul Marchand", "Art O Cathain", "Jerome Wynne", "Philippos Maximos Giavridis", "Stuart Jennings", "Freddy Tuxworth", "Tolga H. Dur", "Sam Deverett", "John Wilkinson", "Jason Gwartz ∗", "Harry Coppock ∗" ], "aliases": [ "https://doi.org/10.48550/arXiv.2603.02277", "https://arxiv.org/abs/2603.02277", "https://arxiv.org/pdf/2603.02277" ] }, { "id": "alignment-faking-author-source", "title": "Alignment faking in large language models", "url": "https://arxiv.org/html/2412.14093", "version": "arXiv:2412.14093v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2412.14093", "authors": [ "Ryan Greenblatt", "† Carson Denison † † footnotemark:", "Benjamin Wright † † footnotemark:", "Fabien Roger † † footnotemark:", "Monte MacDiarmid † † footnotemark:" ], "aliases": [ "https://doi.org/10.48550/arXiv.2412.14093", "https://arxiv.org/abs/2412.14093", "https://arxiv.org/pdf/2412.14093" ] }, { "id": "vct-author-source", "title": "Virology Capabilities Test (VCT): A Multimodal Virology Q&A Benchmark", "url": "https://arxiv.org/html/2504.16137v2", "version": "arXiv:2504.16137v2", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2504.16137", "authors": [ "Jasper Götting a", "Pedro Medeiros a", "Jon G Sanders a", "Nathaniel Li †", "Long Phan", "Karam Elabd", "Lennart Justen", "Dan Hendrycks", "Seth Donoughe ∗" ], "aliases": [ "https://doi.org/10.48550/arXiv.2504.16137", "https://arxiv.org/abs/2504.16137", "https://arxiv.org/pdf/2504.16137" ] }, { "id": "biotier-author-source", "title": "BioTIER: A Refusal Benchmark for Targeted Biological Risk Mitigation", "url": "https://arxiv.org/html/2607.14479v1", "version": "arXiv:2607.14479v1", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2607.14479", "authors": [ "Eleanor M. Marshall", "Pedro Medeiros", "Peter Peneder", "Nelly Mak", "Jacob Kaffey", "Faith Rovenolt", "Mac Walker", "Seth Donoughe", "Jasper Götting" ], "aliases": [ "https://doi.org/10.48550/arXiv.2607.14479", "https://arxiv.org/abs/2607.14479", "https://arxiv.org/pdf/2607.14479" ] }, { "id": "mask-author-source", "title": "The MASK Benchmark: Disentangling Honesty From Accuracy in AI Systems", "url": "https://arxiv.org/html/2503.03750v3", "version": "arXiv:2503.03750v3", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2503.03750", "authors": [ "Richard Ren", "Arunim Agarwal", "Mantas Mazeika", "Cristina Menghini", "Robert Vacareanu", "Brad Kenstler", "Mick Yang", "Isabelle Barrass", "Alice Gatti", "Xuwang Yin", "Eduardo Trevino", "Matias Geralnik", "Adam Khoja", "Dean Lee", "Summer Yue", "Dan Hendrycks" ], "aliases": [ "https://doi.org/10.48550/arXiv.2503.03750", "https://arxiv.org/abs/2503.03750", "https://arxiv.org/pdf/2503.03750" ] }, { "id": "ape-author-source", "title": "It’s the Thought that Counts: Evaluating the Attempts of Frontier LLMs to Persuade on Harmful Topics", "url": "https://arxiv.org/html/2506.02873v4", "version": "arXiv:2506.02873v4", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2506.02873", "authors": [ "\\authorname Matthew Kowal \\authorinstitution FAR.AI \\authorname Jasper Timm \\authorinstitution FAR.AI \\authorname Jean-Francois Godbout \\authorinstitution Université de Montréal, MILA \\authorname Thomas Costello \\authorinstitution Carnegie Mellon University \\authorname Antonio A. Arechar \\authorinstitution MIT, Center for Research and Teaching in Economics \\authorname Gordon Pennycook \\authorinstitution Cornell University, University of Regina \\authorname David Rand \\authorinstitution Cornell University, MIT \\authorname Adam Gleave \\authorinstitution FAR.AI \\authorname Kellin Pelrine \\authorinstitution FAR.AI" ], "aliases": [ "https://doi.org/10.48550/arXiv.2506.02873", "https://arxiv.org/abs/2506.02873", "https://arxiv.org/pdf/2506.02873" ] }, { "id": "sec-bench-publication", "title": "SEC-bench: Automated Benchmarking of LLM Agents on Real-World Software Security Tasks", "url": "https://arxiv.org/abs/2506.11791", "version": "arXiv:2506.11791v2", "publicationDate": "2025-06-13", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2506.11791", "authors": [ "Hwiwon Lee \nZiqi Zhang \nHanxiao Lu † Lingming Zhang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2506.11791", "https://arxiv.org/abs/2506.11791", "https://arxiv.org/pdf/2506.11791" ] }, { "id": "sec-bench-publication-author-source", "title": "SEC-bench: Automated Benchmarking of LLM Agents on Real-World Software Security Tasks", "url": "https://arxiv.org/html/2506.11791", "version": "arXiv:2506.11791v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2506.11791", "authors": [ "Hwiwon Lee \nZiqi Zhang \nHanxiao Lu † Lingming Zhang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2506.11791", "https://arxiv.org/abs/2506.11791", "https://arxiv.org/pdf/2506.11791" ] }, { "id": "exploitbench-publication", "title": "ExploitBench: A Capability Ladder Benchmark for LLM Cybersecurity Agents", "url": "https://arxiv.org/abs/2605.14153", "version": "arXiv:2605.14153v1", "publicationDate": "2026-05-13", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2605.14153", "authors": [ "Seunghyun Lee", "David Brumley" ], "aliases": [ "https://doi.org/10.48550/arXiv.2605.14153", "https://arxiv.org/abs/2605.14153", "https://arxiv.org/pdf/2605.14153" ] }, { "id": "exploitbench-publication-author-source", "title": "ExploitBench : A Capability Ladder Benchmark for LLM Cybersecurity Agents", "url": "https://arxiv.org/html/2605.14153", "version": "arXiv:2605.14153v1 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2605.14153", "authors": [ "Seunghyun Lee", "David Brumley" ], "aliases": [ "https://doi.org/10.48550/arXiv.2605.14153", "https://arxiv.org/abs/2605.14153", "https://arxiv.org/pdf/2605.14153" ] }, { "id": "exploitgym-publication", "title": "ExploitGym: Can AI Agents Turn Security Vulnerabilities into Real Attacks?", "url": "https://rdi.berkeley.edu/blog/exploitgym/", "version": "Source capture 2026-10-25", "publicationDate": "2026-05-13", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://rdi.berkeley.edu/blog/exploitgym", "authors": [ "Zhun Wang", "Nico Schiller", "Hongwei Li", "Srijiith Sesha Narayana", "Milad Nasr", "Nicholas Carlini", "Xiangyu Qi", "Eric Wallace", "Elie Bursztein", "Luca Invernizzi", "Kurt Thomas", "Yan Shoshitaishvili", "Wenbo Guo", "Jingxuan He", "Thorsten Holz", "Dawn Song" ] }, { "id": "exploitgym-publication-author-source", "title": "ExploitGym: Can AI Agents Turn Security Vulnerabilities into Real Attacks?", "url": "https://arxiv.org/html/2605.11086", "version": "arXiv:2605.11086v1 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2605.11086", "authors": [ "Zhun Wang", "Nico Schiller", "Hongwei Li", "Srijiith Sesha Narayana", "Milad Nasr", "Nicholas Carlini", "Xiangyu Qi", "Eric Wallace", "Elie Bursztein", "Luca Invernizzi", "Kurt Thomas", "Yan Shoshitaishvili", "Wenbo Guo", "Jingxuan He", "Thorsten Holz", "Dawn Song" ], "aliases": [ "https://doi.org/10.48550/arXiv.2605.11086", "https://arxiv.org/abs/2605.11086", "https://arxiv.org/pdf/2605.11086" ] }, { "id": "cve-bench-publication", "title": "CVE-Bench: A Benchmark for AI Agents' Ability to Exploit Real-World Web Application Vulnerabilities", "url": "https://arxiv.org/abs/2503.17332", "version": "arXiv:2503.17332v4", "publicationDate": "2025-03-21", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2503.17332", "authors": [ "Yuxuan Zhu", "Antony Kellermann", "Dylan Bowman", "Philip Li", "Akul Gupta", "Adarsh Danda", "Richard Fang", "Conner Jensen", "Eric Ihli", "Jason Benn", "Jet Geronimo", "Avi Dhir", "Sudhit Rao", "Kaicheng Yu", "Twm Stone", "Daniel Kang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2503.17332", "https://arxiv.org/abs/2503.17332", "https://arxiv.org/pdf/2503.17332" ] }, { "id": "cve-bench-publication-author-source", "title": "CVE-Bench: A Benchmark for AI Agents’ Ability to Exploit Real-World\nWeb Application Vulnerabilities", "url": "https://arxiv.org/html/2503.17332", "version": "arXiv:2503.17332v4 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2503.17332", "authors": [ "Yuxuan Zhu", "Antony Kellermann", "Dylan Bowman", "Philip Li", "Akul Gupta", "Adarsh Danda", "Richard Fang", "Conner Jensen", "Eric Ihli", "Jason Benn", "Jet Geronimo", "Avi Dhir", "Sudhit Rao", "Kaicheng Yu", "Twm Stone", "Daniel Kang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2503.17332", "https://arxiv.org/abs/2503.17332", "https://arxiv.org/pdf/2503.17332" ] }, { "id": "sre-bench-publication", "title": "The Next Challenge for Agentic Cybersecurity: A Realistic, Contamination-Free Reverse Engineering Benchmark", "url": "https://arxiv.org/abs/2608.11469", "version": "arXiv:2608.11469v2", "publicationDate": "2026-08-11", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2608.11469", "authors": [ "Jeremy Spence", "Nicholas Assaderaghi", "Feng Xiao", "Jinhao Zhu", "Nikil Ravi", "Xiangyu Qi", "Matthew Jagielski", "Raluca Ada Popa", "Eric Wallace", "Guannan Wei", "Yangruibo Ding", "Zhuo Zhang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2608.11469", "https://arxiv.org/abs/2608.11469", "https://arxiv.org/pdf/2608.11469" ] }, { "id": "sre-bench-publication-author-source", "title": "The Next Challenge for Agentic Cybersecurity: A Realistic, Contamination-Free Reverse Engineering Benchmark", "url": "https://arxiv.org/html/2608.11469", "version": "arXiv:2608.11469v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2608.11469", "authors": [ "Jeremy Spence", "Nicholas Assaderaghi", "Feng Xiao", "Jinhao Zhu", "Nikil Ravi", "Xiangyu Qi", "Matthew Jagielski", "Raluca Ada Popa", "Eric Wallace", "Guannan Wei", "Yangruibo Ding", "Zhuo Zhang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2608.11469", "https://arxiv.org/abs/2608.11469", "https://arxiv.org/pdf/2608.11469" ] }, { "id": "irregular-opus-cyber-assessment-publication", "title": "Assessing Claude Opus 5.5 Against Offensive Security Benchmarks", "url": "https://www.irregular.com/research/assessing-claude-opus-5.5-against-offensive-security-benchmarks", "version": "Source capture 2026-10-25", "publicationDate": "2026-09-22", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.irregular.com/research/assessing-claude-opus-5.5-against-offensive-security-benchmarks" }, { "id": "irregular-astra-cyber-assessment-publication", "title": "Assessing GPT-6 Astra: FrontierCyber Measures a Sharp Increase in Cyber Capability", "url": "https://www.irregular.com/research/assessing-gpt-6-astra", "version": "Source capture 2026-10-25", "publicationDate": "2026-09-03", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.irregular.com/research/assessing-gpt-6-astra" }, { "id": "solve-plus-publication", "title": "Introducing SOLVE+: Scoring Cyber Scenarios (Version 0.5)", "url": "https://www.irregular.com/research/solve-plus", "version": "Source capture 2026-10-25", "publicationDate": "2026-09-03", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.irregular.com/research/solve-plus" }, { "id": "cyberseceval-publication", "title": "Introduction", "url": "https://meta-llama.github.io/PurpleLlama/CyberSecEval/docs/intro", "version": "Source capture 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://meta-llama.github.io/PurpleLlama/CyberSecEval/docs/intro" }, { "id": "cybersoceval-publication", "title": "CyberSOCEval: Benchmarking LLMs Capabilities for Malware Analysis and Threat Intelligence Reasoning", "url": "https://arxiv.org/html/2509.20166v1", "version": "arXiv:2509.20166v1", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2509.20166", "authors": [ "Lauren Deason*†", "Adam Bali†", "Ciprian Bejean‡", "Diana Bolocan*‡", "James Crnkovich*†", "Ioana Croitoru*‡", "Krishna Durai†", "Chase Midler‡", "Calin Miron‡", "David Molnar*†", "Brad Moon‡", "Bruno Ostarcevic‡", "Alberto Peltea‡", "Matt Rosenberg‡", "Catalin Sandu‡", "Arthur Saputkin†", "Sagar Shah*†", "Daniel Stan‡", "Ernest Szocs‡", "Shengye Wan†", "Spencer Whitman†", "Sven Krasser‡", "Joshua Saxe*†" ], "aliases": [ "https://doi.org/10.48550/arXiv.2509.20166", "https://arxiv.org/abs/2509.20166", "https://arxiv.org/pdf/2509.20166" ] }, { "id": "cybersoc-project-source", "title": "CyberSOCEval Benchmark Data", "url": "https://github.com/CrowdStrike/CyberSOCEval_data", "version": "Source capture 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://github.com/CrowdStrike/CyberSOCEval_data" }, { "id": "caisi-glm-cyber-assessment-publication", "title": "CAISI’s Assessment of Z.ai’s GLM-5.3 Cyber Capabilities", "url": "https://www.nist.gov/news-events/news/2026/09/caisis-assessment-zais-glm-53-cyber-capabilities", "version": "Source capture 2026-10-25", "publicationDate": "2026-09-17", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.nist.gov/news-events/news/2026/09/caisis-assessment-zais-glm-53-cyber-capabilities" }, { "id": "hcast-publication", "title": "HCAST: Human-Calibrated Autonomy Software Tasks", "url": "https://arxiv.org/abs/2503.17354", "version": "arXiv:2503.17354v1", "publicationDate": "2025-03-21", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2503.17354", "authors": [ "David Rein", "Joel Becker", "Amy Deng", "Seraphina Nix", "Chris Canal", "Daniel O’Connell", "Pip Arnott", "Ryan Bloom", "Thomas Broadley", "Katharyn Garcia", "Brian Goodrich", "Max Hasin", "Sami Jawhar", "Megan Kinniment", "Thomas Kwa", "Aron Lajko", "Nate Rush", "Lucas Jun Koba Sato", "Sydney Von Arx", "Ben West", "Lawrence Chan", "Elizabeth Barnes" ], "aliases": [ "https://doi.org/10.48550/arXiv.2503.17354", "https://arxiv.org/abs/2503.17354", "https://arxiv.org/pdf/2503.17354" ] }, { "id": "hcast-publication-author-source", "title": "HCAST: Human-Calibrated Autonomy Software Tasks", "url": "https://arxiv.org/html/2503.17354", "version": "arXiv:2503.17354v1 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2503.17354", "authors": [ "David Rein", "Joel Becker", "Amy Deng", "Seraphina Nix", "Chris Canal", "Daniel O’Connell", "Pip Arnott", "Ryan Bloom", "Thomas Broadley", "Katharyn Garcia", "Brian Goodrich", "Max Hasin", "Sami Jawhar", "Megan Kinniment", "Thomas Kwa", "Aron Lajko", "Nate Rush", "Lucas Jun Koba Sato", "Sydney Von Arx", "Ben West", "Lawrence Chan", "Elizabeth Barnes" ], "aliases": [ "https://doi.org/10.48550/arXiv.2503.17354", "https://arxiv.org/abs/2503.17354", "https://arxiv.org/pdf/2503.17354" ] }, { "id": "mirrorcode-publication", "title": "MirrorCode: Evidence that AI can already do some weeks-long coding tasks", "url": "https://metr.org/blog/2026-04-10-mirrorcode-preliminary-results/", "version": "Source capture 2026-10-25", "publicationDate": "2026-04-10", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://metr.org/blog/2026-04-10-mirrorcode-preliminary-results" }, { "id": "expenditure-horizon-publication", "title": "Expenditure Horizon: Measuring Optimization Ability, with an Application to NanoGPT", "url": "https://metr.org/blog/2026-07-21-expenditure-horizon/", "version": "Source capture 2026-10-25", "publicationDate": "2026-07-21", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://metr.org/blog/2026-07-21-expenditure-horizon" }, { "id": "mle-bench-publication", "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "url": "https://arxiv.org/abs/2410.07095", "version": "arXiv:2410.07095v6", "publicationDate": "2024-10-09", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2410.07095", "authors": [ "Chan Jun Shern *", "Neil Chowdhury", "Oliver Jaffe *", "James Aung *", "Dane Sherburn *", "Evan Mays *", "Giulio Starace *", "Kevin Liu", "Leon Maksin", "Tejal Patwardhan", "Lilian Weng", "Aleksander MądryOpenAI" ], "aliases": [ "https://doi.org/10.48550/arXiv.2410.07095", "https://arxiv.org/abs/2410.07095", "https://arxiv.org/pdf/2410.07095" ] }, { "id": "mle-bench-publication-author-source", "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "url": "https://arxiv.org/html/2410.07095", "version": "arXiv:2410.07095v6 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2410.07095", "authors": [ "Chan Jun Shern *", "Neil Chowdhury", "Oliver Jaffe *", "James Aung *", "Dane Sherburn *", "Evan Mays *", "Giulio Starace *", "Kevin Liu", "Leon Maksin", "Tejal Patwardhan", "Lilian Weng", "Aleksander MądryOpenAI" ], "aliases": [ "https://doi.org/10.48550/arXiv.2410.07095", "https://arxiv.org/abs/2410.07095", "https://arxiv.org/pdf/2410.07095" ] }, { "id": "posttrainbench-publication", "title": "PostTrainBench: Can LLM Agents Automate LLM Post-Training?", "url": "https://arxiv.org/abs/2603.08640", "version": "arXiv:2603.08640v2", "publicationDate": "2026-03-09", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2603.08640", "authors": [ "Ben Rank ∗", "Hardik Bhatnagar ∗", "Ameya Prabhu", "Shira Eisenberg †", "Nguyen Karina", "Matthias Bethge", "Maksym Andriushchenko" ], "aliases": [ "https://doi.org/10.48550/arXiv.2603.08640", "https://arxiv.org/abs/2603.08640", "https://arxiv.org/pdf/2603.08640" ] }, { "id": "posttrainbench-publication-author-source", "title": "PostTrainBench : Can LLM Agents Automate LLM Post-Training?", "url": "https://arxiv.org/html/2603.08640", "version": "arXiv:2603.08640v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2603.08640", "authors": [ "Ben Rank ∗", "Hardik Bhatnagar ∗", "Ameya Prabhu", "Shira Eisenberg †", "Nguyen Karina", "Matthias Bethge", "Maksym Andriushchenko" ], "aliases": [ "https://doi.org/10.48550/arXiv.2603.08640", "https://arxiv.org/abs/2603.08640", "https://arxiv.org/pdf/2603.08640" ] }, { "id": "deepmind-dangerous-capabilities-publication", "title": "Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/abs/2403.13793", "version": "arXiv:2403.13793v2", "publicationDate": "2024-03-20", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2403.13793", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Grégoire Delétang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "aliases": [ "https://doi.org/10.48550/arXiv.2403.13793", "https://arxiv.org/abs/2403.13793", "https://arxiv.org/pdf/2403.13793" ] }, { "id": "deepmind-dangerous-capabilities-publication-author-source", "title": "Evaluating Frontier Models for Dangerous Capabilities", "url": "https://arxiv.org/html/2403.13793", "version": "arXiv:2403.13793 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2403.13793", "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Grégoire Delétang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "aliases": [ "https://doi.org/10.48550/arXiv.2403.13793", "https://arxiv.org/abs/2403.13793", "https://arxiv.org/pdf/2403.13793" ] }, { "id": "deepmind-capability-author-source", "title": "Evaluating Frontier Models for Dangerous Capabilities", "url": "https://deepmind.google/research/publications/78150/", "version": "Source capture 2026-10-25", "publicationDate": "2024-03-21", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/research/publications/78150" }, { "id": "self-replication-extended-publication", "title": "Large language model-powered AI systems achieve self-replication with no human intervention", "url": "https://arxiv.org/abs/2503.17378", "version": "arXiv:2503.17378v2", "publicationDate": "2025-03-14", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2503.17378", "authors": [ "Xudong Pan † {\\dagger} (潘旭东)", "Jiarun Dai † {\\dagger} (戴嘉润)", "Yihe Fan (范一禾)", "Minyuan Luo (罗铭源)", "Changyi Li (李长艺)", "Min Yang ∗ (杨珉)" ], "aliases": [ "https://doi.org/10.48550/arXiv.2503.17378", "https://arxiv.org/abs/2503.17378", "https://arxiv.org/pdf/2503.17378" ] }, { "id": "self-replication-extended-publication-author-source", "title": "Large language model-powered AI systems achieve self-replication with no human intervention", "url": "https://arxiv.org/html/2503.17378", "version": "arXiv:2503.17378 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2503.17378", "authors": [ "Xudong Pan † {\\dagger} (潘旭东)", "Jiarun Dai † {\\dagger} (戴嘉润)", "Yihe Fan (范一禾)", "Minyuan Luo (罗铭源)", "Changyi Li (李长艺)", "Min Yang ∗ (杨珉)" ], "aliases": [ "https://doi.org/10.48550/arXiv.2503.17378", "https://arxiv.org/abs/2503.17378", "https://arxiv.org/pdf/2503.17378" ] }, { "id": "palisade-network-replication-publication", "title": "Language Models Can Autonomously Hack and Self-Replicate", "url": "https://arxiv.org/abs/2605.06760", "version": "arXiv:2605.06760v1", "publicationDate": "2026-05-07", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2605.06760", "authors": [ "Air, Alena", "Reworr", "Kotov, Nikolaj", "Volkov, Dmitrii", "Steidley, John", "Ladish, Jeffrey" ], "aliases": [ "https://doi.org/10.48550/arXiv.2605.06760", "https://arxiv.org/abs/2605.06760", "https://arxiv.org/pdf/2605.06760" ] }, { "id": "palisade-replication-project", "title": "Language Models Can Autonomously Hack and Self-Replicate", "url": "https://palisaderesearch.org/research/self-replication", "version": "Source capture 2026-10-25", "publicationDate": "2026-05-07", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://palisaderesearch.org/research/self-replication" }, { "id": "agentic-self-modification-publication", "title": "Agentic Self-Modification in Open-Weights Systems", "url": "https://www.irregular.com/research/agentic-self-modification-in-open-weights-systems", "version": "Source capture 2026-10-25", "publicationDate": "2026-09-16", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.irregular.com/research/agentic-self-modification-in-open-weights-systems" }, { "id": "machiavelli-publication", "title": "Do the Rewards Justify the Means? Measuring Trade-Offs Between Rewards and Ethical Behavior in the MACHIAVELLI Benchmark", "url": "https://arxiv.org/abs/2304.03279", "version": "arXiv:2304.03279v4", "publicationDate": "2023-04-06", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2304.03279", "authors": [ "Alexander Pan", "Jun Shern Chan", "Andy Zou", "Nathaniel Li", "Steven Basart", "Thomas Woodside", "Jonathan Ng", "Hanlin Zhang", "Scott Emmons", "Dan Hendrycks" ], "aliases": [ "https://doi.org/10.48550/arXiv.2304.03279", "https://arxiv.org/abs/2304.03279", "https://arxiv.org/pdf/2304.03279" ] }, { "id": "machiavelli-publication-author-source", "title": "Do the Rewards Justify the Means? Measuring Trade-Offs Between Rewards and Ethical Behavior in the Machiavelli Benchmark", "url": "https://arxiv.org/html/2304.03279", "version": "arXiv:2304.03279v4 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2304.03279", "authors": [ "Alexander Pan", "Jun Shern Chan", "Andy Zou", "Nathaniel Li", "Steven Basart", "Thomas Woodside", "Jonathan Ng", "Hanlin Zhang", "Scott Emmons", "Dan Hendrycks" ], "aliases": [ "https://doi.org/10.48550/arXiv.2304.03279", "https://arxiv.org/abs/2304.03279", "https://arxiv.org/pdf/2304.03279" ] }, { "id": "emergent-misalignment-publication", "title": "Emergent Misalignment: Narrow finetuning can produce broadly misaligned LLMs", "url": "https://truthful.ai/papers/emergent-misalignment/", "version": "Source capture 2026-10-25", "publicationDate": "2025-02-25", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://truthful.ai/papers/emergent-misalignment", "authors": [ "Jan Betley", "Daniel Tan", "Niels Warncke", "Anna Sztyber-Betley", "Xuchan Bao", "Martín Soto", "Nathan Labenz", "Owain Evans" ] }, { "id": "emergent-misalignment-publication-author-source", "title": "Emergent Misalignment: Narrow finetuning can produce broadly misaligned LLMs 1 This paper contains model-generated content that might be offensive. 1", "url": "https://arxiv.org/html/2502.17424", "version": "arXiv:2502.17424 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2502.17424", "authors": [ "Jan Betley", "Daniel Tan", "Niels Warncke", "Anna Sztyber-Betley", "Xuchan Bao", "Martín Soto", "Nathan Labenz", "Owain Evans" ], "aliases": [ "https://doi.org/10.48550/arXiv.2502.17424", "https://arxiv.org/abs/2502.17424", "https://arxiv.org/pdf/2502.17424" ] }, { "id": "school-of-reward-hacks-publication", "title": "School of Reward Hacks: Hacking harmless tasks generalizes to misaligned behavior in LLMs", "url": "https://truthful.ai/papers/school-of-reward-hacks/", "version": "Source capture 2026-10-25", "publicationDate": "2025-08-25", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://truthful.ai/papers/school-of-reward-hacks", "authors": [ "Mia Taylor", "James Chua", "Jan Betley", "Johannes Treutlein", "Owain Evans" ] }, { "id": "school-of-reward-hacks-publication-author-source", "title": "School of reward hacks: Hacking harmless tasks generalizes to misaligned behavior in LLMs", "url": "https://arxiv.org/html/2508.17511", "version": "arXiv:2508.17511 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2508.17511", "authors": [ "Mia Taylor", "James Chua", "Jan Betley", "Johannes Treutlein", "Owain Evans" ], "aliases": [ "https://doi.org/10.48550/arXiv.2508.17511", "https://arxiv.org/abs/2508.17511", "https://arxiv.org/pdf/2508.17511" ] }, { "id": "conditional-misalignment-publication", "title": "Conditional Misalignment: Common Interventions Can Hide Emergent Misalignment Behind Contextual Triggers", "url": "https://truthful.ai/papers/conditional-misalignment/", "version": "Source capture 2026-10-25", "publicationDate": "2026-04-28", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://truthful.ai/papers/conditional-misalignment", "authors": [ "Jan Dubiński", "Jan Betley", "Anna Sztyber-Betley", "Daniel C.H. Tan", "Owain Evans" ] }, { "id": "conditional-misalignment-publication-author-source", "title": "Conditional misalignment: common interventions can hide emergent misalignment behind contextual triggers", "url": "https://arxiv.org/html/2604.25891", "version": "arXiv:2604.25891 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2604.25891", "authors": [ "Jan Dubiński", "Jan Betley", "Anna Sztyber-Betley", "Daniel C.H. Tan", "Owain Evans" ], "aliases": [ "https://doi.org/10.48550/arXiv.2604.25891", "https://arxiv.org/abs/2604.25891", "https://arxiv.org/pdf/2604.25891" ] }, { "id": "reward-hacking-emergent-misalignment-publication", "title": "From shortcuts to sabotage : Natural emergent misalignment from reward hacking", "url": "https://www.anthropic.com/research/emergent-misalignment-reward-hacking", "version": "Source capture 2026-10-25", "publicationDate": "2025-11-21", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/research/emergent-misalignment-reward-hacking" }, { "id": "contrastive-reward-seeking-publication", "title": "Measuring Reward-Seeking via Contrastive Belief Updates", "url": "https://arxiv.org/abs/2607.18966", "version": "arXiv:2607.18966v1", "publicationDate": "2026-07-21", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2607.18966", "authors": [ "Axel Højmark", "Jérémy Scheurer 1 1 footnotemark: 1", "Evgenia Nitishinskaya", "Felix Hofstätter 1 1 footnotemark: 1", "Jason Wolfe 2 2 footnotemark: 2", "Theodore Ehrenborg 1 1 footnotemark: 1", "Bronson Schoen 1 1 footnotemark: 1", "Alexander Meinke 1 1 footnotemark: 1", "Apollo Research & OpenAI" ], "aliases": [ "https://doi.org/10.48550/arXiv.2607.18966", "https://arxiv.org/abs/2607.18966", "https://arxiv.org/pdf/2607.18966" ] }, { "id": "contrastive-reward-seeking-publication-author-source", "title": "Measuring Reward-Seeking via Contrastive Belief Updates", "url": "https://arxiv.org/html/2607.18966", "version": "arXiv:2607.18966 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2607.18966", "authors": [ "Axel Højmark", "Jérémy Scheurer 1 1 footnotemark: 1", "Evgenia Nitishinskaya", "Felix Hofstätter 1 1 footnotemark: 1", "Jason Wolfe 2 2 footnotemark: 2", "Theodore Ehrenborg 1 1 footnotemark: 1", "Bronson Schoen 1 1 footnotemark: 1", "Alexander Meinke 1 1 footnotemark: 1", "Apollo Research & OpenAI" ], "aliases": [ "https://doi.org/10.48550/arXiv.2607.18966", "https://arxiv.org/abs/2607.18966", "https://arxiv.org/pdf/2607.18966" ] }, { "id": "sabotage-safety-research-publication", "title": "Evaluating whether AI models would sabotage AI safety research", "url": "https://www.aisi.gov.uk/research/evaluating-whether-ai-models-would-sabotage-ai-safety-research", "version": "Source capture 2026-10-25", "publicationDate": "2026-04-27", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/evaluating-whether-ai-models-would-sabotage-ai-safety-research", "authors": [ "Robert Kirk", "Alexandra Souly", "Kai Fronsdal", "Abby D’Cruz", "Xander Davies" ] }, { "id": "sabotage-safety-research-publication-author-source", "title": "Evaluating whether ai models would sabotage ai safety research", "url": "https://arxiv.org/html/2604.24618", "version": "arXiv:2604.24618 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2604.24618", "authors": [ "Robert Kirk", "Alexandra Souly", "Kai Fronsdal", "Abby D’Cruz", "Xander Davies" ], "aliases": [ "https://doi.org/10.48550/arXiv.2604.24618", "https://arxiv.org/abs/2604.24618", "https://arxiv.org/pdf/2604.24618" ] }, { "id": "ai-sandbagging-publication", "title": "AI Sandbagging: Language Models can Strategically Underperform on Evaluations", "url": "https://arxiv.org/abs/2406.07358", "version": "arXiv:2406.07358v4", "publicationDate": "2024-06-11", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2406.07358", "authors": [ "Teun van der Weij", "MATS", "Felix Hofstätter ∗", "Oliver Jaffe ∗", "Samuel F. Brown", "Francis Rhys Ward" ], "aliases": [ "https://doi.org/10.48550/arXiv.2406.07358", "https://arxiv.org/abs/2406.07358", "https://arxiv.org/pdf/2406.07358" ] }, { "id": "ai-sandbagging-publication-author-source", "title": "AI Sandbagging: Language Models can Strategically Underperform on Evaluations", "url": "https://arxiv.org/html/2406.07358", "version": "arXiv:2406.07358 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2406.07358", "authors": [ "Teun van der Weij", "MATS", "Felix Hofstätter ∗", "Oliver Jaffe ∗", "Samuel F. Brown", "Francis Rhys Ward" ], "aliases": [ "https://doi.org/10.48550/arXiv.2406.07358", "https://arxiv.org/abs/2406.07358", "https://arxiv.org/pdf/2406.07358" ] }, { "id": "sleeper-agents-publication", "title": "Sleeper Agents : Training deceptive LLMs that persist through safety training", "url": "https://www.anthropic.com/research/sleeper-agents-training-deceptive-llms-that-persist-through-safety-training", "version": "Source capture 2026-10-25", "publicationDate": "2024-01-14", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/research/sleeper-agents-training-deceptive-llms-that-persist-through-safety-training", "authors": [ "Evan Hubinger, Carson Denison † † footnotemark: , Jesse Mu † † footnotemark: , Mike Lambert † † footnotemark: , Meg Tong, Monte MacDiarmid,Tamera Lanham, Daniel M. Ziegler, Tim Maxwell, Newton ChengAdam Jermyn, Amanda Askell, Ansh Radhakrishnan, Cem Anil, David Duvenaud", "Deep Ganguli", "Fazl Barez", "Jack Clark", "Kamal Ndousse", "Kshitij Sachan", "Michael Sellitto Mrinank Sharma, Nova DasSarma, Roger Grosse, Shauna Kravec, Yuntao Bai, Zachary Witten", "Marina Favaro", "Jan Brauner ∘", "Holden Karnofsky", "Paul Christiano ⋄", "Samuel R. Bowman Logan Graham, Jared Kaplan, Sören Mindermann ‡∘ , Ryan Greenblatt † , Buck Shlegeris † ,Nicholas Schiefer † † footnotemark: , Ethan Perez † † footnotemark: Anthropic, † Redwood Research, ‡ Mila Quebec AI Institute, ∘ University of Oxford,", "⋄ Alignment Research Center", "Open Philanthropy", "Apart Research" ] }, { "id": "sleeper-agents-publication-author-source", "title": "Sleeper Agents: Training Deceptive LLMs that Persist Through Safety Training", "url": "https://arxiv.org/html/2401.05566", "version": "arXiv:2401.05566 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2401.05566", "authors": [ "Evan Hubinger, Carson Denison † † footnotemark: , Jesse Mu † † footnotemark: , Mike Lambert † † footnotemark: , Meg Tong, Monte MacDiarmid,Tamera Lanham, Daniel M. Ziegler, Tim Maxwell, Newton ChengAdam Jermyn, Amanda Askell, Ansh Radhakrishnan, Cem Anil, David Duvenaud", "Deep Ganguli", "Fazl Barez", "Jack Clark", "Kamal Ndousse", "Kshitij Sachan", "Michael Sellitto Mrinank Sharma, Nova DasSarma, Roger Grosse, Shauna Kravec, Yuntao Bai, Zachary Witten", "Marina Favaro", "Jan Brauner ∘", "Holden Karnofsky", "Paul Christiano ⋄", "Samuel R. Bowman Logan Graham, Jared Kaplan, Sören Mindermann ‡∘ , Ryan Greenblatt † , Buck Shlegeris † ,Nicholas Schiefer † † footnotemark: , Ethan Perez † † footnotemark: Anthropic, † Redwood Research, ‡ Mila Quebec AI Institute, ∘ University of Oxford,", "⋄ Alignment Research Center", "Open Philanthropy", "Apart Research" ], "aliases": [ "https://doi.org/10.48550/arXiv.2401.05566", "https://arxiv.org/abs/2401.05566", "https://arxiv.org/pdf/2401.05566" ] }, { "id": "subliminal-learning-publication", "title": "Subliminal Learning: Language models transmit behavioral traits via hidden signals in data", "url": "https://truthful.ai/papers/subliminal-learning/", "version": "Source capture 2026-10-25", "publicationDate": "2025-07-20", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://truthful.ai/papers/subliminal-learning", "authors": [ "Alex Cloud", "Minh Le", "James Chua", "Jan Betley", "Anna Sztyber-Betley", "Sören Mindermann", "Jacob Hilton", "Samuel Marks", "Owain Evans" ] }, { "id": "ai-control-publication", "title": "AI Control: Improving Safety Despite Intentional Subversion", "url": "https://arxiv.org/abs/2312.06942", "version": "arXiv:2312.06942v5", "publicationDate": "2023-12-12", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2312.06942", "authors": [ "Ryan Greenblatt ∗ Buck Shlegeris      Kshitij Sachan      Fabien Roger Redwood Research" ], "aliases": [ "https://doi.org/10.48550/arXiv.2312.06942", "https://arxiv.org/abs/2312.06942", "https://arxiv.org/pdf/2312.06942" ] }, { "id": "ai-control-publication-author-source", "title": "AI Control: Improving Safety Despite Intentional Subversion", "url": "https://arxiv.org/html/2312.06942", "version": "arXiv:2312.06942 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2312.06942", "authors": [ "Ryan Greenblatt ∗ Buck Shlegeris      Kshitij Sachan      Fabien Roger Redwood Research" ], "aliases": [ "https://doi.org/10.48550/arXiv.2312.06942", "https://arxiv.org/abs/2312.06942", "https://arxiv.org/pdf/2312.06942" ] }, { "id": "ctrl-z-publication", "title": "Ctrl-Z: Controlling AI Agents via Resampling", "url": "https://arxiv.org/abs/2504.10374", "version": "arXiv:2504.10374v1", "publicationDate": "2025-04-14", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2504.10374", "authors": [ "Aryan Bhatt", "Cody Rushing", "Adam Kaufman", "Tyler Tracy", "Vasil Georgiev", "David Matolcsi", "Akbir Khan", "Buck Shlegeris" ], "aliases": [ "https://doi.org/10.48550/arXiv.2504.10374", "https://arxiv.org/abs/2504.10374", "https://arxiv.org/pdf/2504.10374" ] }, { "id": "ctrl-z-publication-author-source", "title": "Ctrl-Z: Controlling AI Agents via Resampling", "url": "https://arxiv.org/html/2504.10374", "version": "arXiv:2504.10374 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2504.10374", "authors": [ "Aryan Bhatt", "Cody Rushing", "Adam Kaufman", "Tyler Tracy", "Vasil Georgiev", "David Matolcsi", "Akbir Khan", "Buck Shlegeris" ], "aliases": [ "https://doi.org/10.48550/arXiv.2504.10374", "https://arxiv.org/abs/2504.10374", "https://arxiv.org/pdf/2504.10374" ] }, { "id": "basharena-publication", "title": "BashArena: A Control Setting for Highly Privileged AI Agents", "url": "https://arxiv.org/abs/2512.15688", "version": "arXiv:2512.15688v1", "publicationDate": "2025-12-17", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2512.15688", "authors": [ "Adam Kaufman *", "James Lucassen *" ], "aliases": [ "https://doi.org/10.48550/arXiv.2512.15688", "https://arxiv.org/abs/2512.15688", "https://arxiv.org/pdf/2512.15688" ] }, { "id": "basharena-publication-author-source", "title": "BashArena: A Control Setting for Highly Privileged AI Agents", "url": "https://arxiv.org/html/2512.15688", "version": "arXiv:2512.15688v1 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2512.15688", "authors": [ "Adam Kaufman *", "James Lucassen *" ], "aliases": [ "https://doi.org/10.48550/arXiv.2512.15688", "https://arxiv.org/abs/2512.15688", "https://arxiv.org/pdf/2512.15688" ] }, { "id": "multi-agent-control-publication", "title": "Multi-Agent AI Control: Distributed Attacks Hamper Per-Instance Monitors", "url": "https://www.aisi.gov.uk/research/multi-agent-ai-control-distributed-attacks-hamper-per-instance-monitors", "version": "Source capture 2026-10-25", "publicationDate": "2026-07-08", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/multi-agent-ai-control-distributed-attacks-hamper-per-instance-monitors", "authors": [ "Oliver Makins", "Orazio Angelini 1 1 footnotemark: 1", "Zohreh Shams", "Mary Phuong" ] }, { "id": "multi-agent-control-publication-author-source", "title": "Multi-Agent AI Control: Distributed Attacks Hamper Per-Instance Monitors", "url": "https://arxiv.org/html/2607.07368", "version": "arXiv:2607.07368 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2607.07368", "authors": [ "Oliver Makins", "Orazio Angelini 1 1 footnotemark: 1", "Zohreh Shams", "Mary Phuong" ], "aliases": [ "https://doi.org/10.48550/arXiv.2607.07368", "https://arxiv.org/abs/2607.07368", "https://arxiv.org/pdf/2607.07368" ] }, { "id": "anti-scheming-training-publication", "title": "Stress Testing Deliberative Alignment for Anti-Scheming Training", "url": "https://www.apolloresearch.ai/science/stress-testing-deliberative-alignment-for-anti-scheming-training", "version": "Source capture 2026-10-25", "publicationDate": "2025-09-17", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.apolloresearch.ai/science/stress-testing-deliberative-alignment-for-anti-scheming-training" }, { "id": "malt-publication", "title": "MALT: A Dataset of Natural and Prompted Behaviors That Threaten Eval Integrity", "url": "https://metr.org/blog/2025-10-14-malt-dataset-of-natural-and-prompted-behaviors/", "version": "Source capture 2026-10-25", "publicationDate": "2025-10-14", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://metr.org/blog/2025-10-14-malt-dataset-of-natural-and-prompted-behaviors" }, { "id": "metr-monitorability-publication", "title": "Early work on monitorability evaluations", "url": "https://metr.org/blog/2026-01-19-early-work-on-monitorability-evaluations/", "version": "Source capture 2026-10-25", "publicationDate": "2026-01-22", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://metr.org/blog/2026-01-19-early-work-on-monitorability-evaluations" }, { "id": "auditing-games-publication", "title": "Auditing Games for Sandbagging", "url": "https://arxiv.org/abs/2512.07810", "version": "arXiv:2512.07810v1", "publicationDate": "2025-12-08", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2512.07810", "authors": [ "Jordan Taylor", "Sid Black Dillon Bowen Thomas Read Satvik Golechha", "Alex Zelenka-Martin Oliver Makins Connor Kissane Kola Ayonrinde Jacob Merizian", "Samuel Marks Chris Cundy Joseph Bloom" ], "aliases": [ "https://doi.org/10.48550/arXiv.2512.07810", "https://arxiv.org/abs/2512.07810", "https://arxiv.org/pdf/2512.07810" ] }, { "id": "auditing-games-publication-author-source", "title": "Auditing Games for Sandbagging", "url": "https://arxiv.org/html/2512.07810", "version": "arXiv:2512.07810 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2512.07810", "authors": [ "Jordan Taylor", "Sid Black Dillon Bowen Thomas Read Satvik Golechha", "Alex Zelenka-Martin Oliver Makins Connor Kissane Kola Ayonrinde Jacob Merizian", "Samuel Marks Chris Cundy Joseph Bloom" ], "aliases": [ "https://doi.org/10.48550/arXiv.2512.07810", "https://arxiv.org/abs/2512.07810", "https://arxiv.org/pdf/2512.07810" ] }, { "id": "belief-verified-lie-detectors-publication", "title": "\"Did you lie?\": Evaluating Lie Detectors across Model Scale and Belief-Verified Model Organisms", "url": "https://www.aisi.gov.uk/research/did-you-lie-evaluating-lie-detectors-across-model-scale-and-belief-verified-model-organisms", "version": "Source capture 2026-10-25", "publicationDate": "2026-06-17", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/did-you-lie-evaluating-lie-detectors-across-model-scale-and-belief-verified-model-organisms", "authors": [ "Alan Cooney", "David Africa", "Geoffrey Irving" ] }, { "id": "belief-verified-lie-detectors-publication-author-source", "title": "“Did you lie?” Evaluating Lie Detectors across Model Scale and Belief-Verified Model Organisms", "url": "https://arxiv.org/html/2606.12618", "version": "arXiv:2606.12618 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2606.12618", "authors": [ "Alan Cooney", "David Africa", "Geoffrey Irving" ], "aliases": [ "https://doi.org/10.48550/arXiv.2606.12618", "https://arxiv.org/abs/2606.12618", "https://arxiv.org/pdf/2606.12618" ] }, { "id": "obfuscation-atlas-publication", "title": "The Obfuscation Atlas: Mapping Where Honesty Emerges in RLVR with Deception Probes", "url": "https://www.far.ai/research/the-obfuscation-atlas-mapping-where-honesty-emerges-in-rlvr-with-deception-probes", "version": "Source capture 2026-10-25", "publicationDate": "2026-02-16", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.far.ai/research/the-obfuscation-atlas-mapping-where-honesty-emerges-in-rlvr-with-deception-probes" }, { "id": "tamperbench-publication", "title": "TamperBench: Systematically Stress-Testing LLM Safety Under Fine-Tuning and Tampering", "url": "https://arxiv.org/abs/2602.06911", "version": "arXiv:2602.06911v2", "publicationDate": "2026-02-06", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2602.06911", "authors": [ "Saad Hossain", "Tom Tseng", "Punya Syon Pandey", "Samanvay Vajpayee", "Matthew Kowal", "Nayeema Nonta", "Samuel Simko", "Stephen Casper", "Zhijing Jin", "Kellin Pelrine", "Sirisha Rambhatla" ], "aliases": [ "https://doi.org/10.48550/arXiv.2602.06911", "https://arxiv.org/abs/2602.06911", "https://arxiv.org/pdf/2602.06911" ] }, { "id": "tamperbench-publication-author-source", "title": "TamperBench : Systematically Stress-Testing LLM Safety Under Fine-Tuning and Tampering", "url": "https://arxiv.org/html/2602.06911", "version": "arXiv:2602.06911v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2602.06911", "authors": [ "Saad Hossain", "Tom Tseng", "Punya Syon Pandey", "Samanvay Vajpayee", "Matthew Kowal", "Nayeema Nonta", "Samuel Simko", "Stephen Casper", "Zhijing Jin", "Kellin Pelrine", "Sirisha Rambhatla" ], "aliases": [ "https://doi.org/10.48550/arXiv.2602.06911", "https://arxiv.org/abs/2602.06911", "https://arxiv.org/pdf/2602.06911" ] }, { "id": "automated-alignment-mitigation-publication", "title": "Automated researchers can reliably mitigate alignment failures", "url": "https://www.anthropic.com/research/automated-researchers-mitigate-alignment-failures", "version": "Source capture 2026-10-25", "publicationDate": "2026-08-28", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/research/automated-researchers-mitigate-alignment-failures" }, { "id": "training-run-evaluations-publication", "title": "We need 3rd party Training-Run Evaluations", "url": "https://www.apolloresearch.ai/science/we-need-3rd-party-training-run-assessments", "version": "Source capture 2026-10-25", "publicationDate": "2026-07-05", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.apolloresearch.ai/science/we-need-3rd-party-training-run-assessments" }, { "id": "cot-faithfulness-publication", "title": "Reasoning models don't always say what they think", "url": "https://www.anthropic.com/research/reasoning-models-dont-say-think", "version": "Source capture 2026-10-25", "publicationDate": "2025-04-03", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/research/reasoning-models-dont-say-think" }, { "id": "cot-misbehavior-monitoring-publication", "title": "Monitoring Reasoning Models for Misbehavior and the Risks of Promoting Obfuscation", "url": "https://arxiv.org/abs/2503.11926", "version": "arXiv:2503.11926v1", "publicationDate": "2025-03-14", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2503.11926", "authors": [ "Bowen Baker Joost Huizinga *† Leo Gao † Zehao Dou † Melody Y. Guan † Aleksander Madry † Wojciech Zaremba † Jakub Pachocki † David Farhi *†" ], "aliases": [ "https://doi.org/10.48550/arXiv.2503.11926", "https://arxiv.org/abs/2503.11926", "https://arxiv.org/pdf/2503.11926" ] }, { "id": "cot-misbehavior-monitoring-publication-author-source", "title": "Monitoring Reasoning Models for Misbehavior and the Risks of Promoting Obfuscation", "url": "https://arxiv.org/html/2503.11926", "version": "arXiv:2503.11926 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2503.11926", "authors": [ "Bowen Baker Joost Huizinga *† Leo Gao † Zehao Dou † Melody Y. Guan † Aleksander Madry † Wojciech Zaremba † Jakub Pachocki † David Farhi *†" ], "aliases": [ "https://doi.org/10.48550/arXiv.2503.11926", "https://arxiv.org/abs/2503.11926", "https://arxiv.org/pdf/2503.11926" ] }, { "id": "injecagent-publication", "title": "InjecAgent: Benchmarking Indirect Prompt Injections in Tool-Integrated Large Language Model Agents", "url": "https://arxiv.org/abs/2403.02691", "version": "arXiv:2403.02691v3", "publicationDate": "2024-03-05", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2403.02691", "authors": [ "Qiusi Zhan", "Zhixiang Liang", "Zifan Ying", "Daniel Kang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2403.02691", "https://arxiv.org/abs/2403.02691", "https://arxiv.org/pdf/2403.02691" ] }, { "id": "injecagent-publication-author-source", "title": "InjecAgent : Benchmarking Indirect Prompt Injections in Tool-Integrated Large Language Model Agents", "url": "https://arxiv.org/html/2403.02691", "version": "arXiv:2403.02691v3 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2403.02691", "authors": [ "Qiusi Zhan", "Zhixiang Liang", "Zifan Ying", "Daniel Kang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2403.02691", "https://arxiv.org/abs/2403.02691", "https://arxiv.org/pdf/2403.02691" ] }, { "id": "agent-safetybench-publication", "title": "Agent-SafetyBench: Evaluating the Safety of LLM Agents", "url": "https://arxiv.org/abs/2412.14470", "version": "arXiv:2412.14470v2", "publicationDate": "2024-12-19", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2412.14470", "authors": [ "Zhexin Zhang 1 1 1 Equal contribution.", "Shiyao Cui 1 1 1 Equal contribution.", "Yida Lu 1 1 1 Equal contribution.", "Jingzhuo Zhou 1 1 1 Equal contribution.", "Junxiao Yang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2412.14470", "https://arxiv.org/abs/2412.14470", "https://arxiv.org/pdf/2412.14470" ] }, { "id": "agent-safetybench-publication-author-source", "title": "Agent-SafetyBench : Evaluating the Safety of LLM Agents", "url": "https://arxiv.org/html/2412.14470", "version": "arXiv:2412.14470v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2412.14470", "authors": [ "Zhexin Zhang 1 1 1 Equal contribution.", "Shiyao Cui 1 1 1 Equal contribution.", "Yida Lu 1 1 1 Equal contribution.", "Jingzhuo Zhou 1 1 1 Equal contribution.", "Junxiao Yang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2412.14470", "https://arxiv.org/abs/2412.14470", "https://arxiv.org/pdf/2412.14470" ] }, { "id": "petri-publication", "title": "Petri : An open-source auditing tool to accelerate AI safety research", "url": "https://www.anthropic.com/research/petri-open-source-auditing", "version": "Source capture 2026-10-25", "publicationDate": "2025-10-06", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/research/petri-open-source-auditing" }, { "id": "bloom-publication", "title": "Introducing Bloom : an open source tool for automated behavioral evaluations", "url": "https://www.anthropic.com/research/bloom", "version": "Source capture 2026-10-25", "publicationDate": "2025-12-19", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/research/bloom" }, { "id": "truthfulqa-publication", "title": "TruthfulQA: Measuring how models mimic human falsehoods", "url": "https://truthful.ai/papers/truthfulqa/", "version": "Source capture 2026-10-25", "publicationDate": "2021-09-08", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://truthful.ai/papers/truthfulqa", "authors": [ "Stephanie Lin", "Jacob Hilton", "Owain Evans" ] }, { "id": "truthfulqa-publication-author-source", "title": "TruthfulQA: Measuring How Models Mimic Human Falsehoods", "url": "https://arxiv.org/html/2109.07958", "version": "arXiv:2109.07958v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2109.07958", "authors": [ "Stephanie Lin", "Jacob Hilton", "Owain Evans" ], "aliases": [ "https://doi.org/10.48550/arXiv.2109.07958", "https://arxiv.org/abs/2109.07958", "https://arxiv.org/pdf/2109.07958" ] }, { "id": "value-leakage-publication", "title": "Value Leakage: An LLM's Answers Are Silently Shaped by Its Own Values", "url": "https://truthful.ai/papers/value-leakage/", "version": "Source capture 2026-10-25", "publicationDate": "2026-07-15", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://truthful.ai/papers/value-leakage", "authors": [ "Jan Betley", "Johannes Treutlein † † footnotemark:", "Jan Dubiński", "Harry Mayne", "Karol Gałązka", "Niels Warncke", "Anna Sztyber-Betley", "Owain Evans" ] }, { "id": "value-leakage-publication-author-source", "title": "Value Leakage: An LLM’s Answers Are Silently Shaped by Its Own Values", "url": "https://arxiv.org/html/2607.14345", "version": "arXiv:2607.14345 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2607.14345", "authors": [ "Jan Betley", "Johannes Treutlein † † footnotemark:", "Jan Dubiński", "Harry Mayne", "Karol Gałązka", "Niels Warncke", "Anna Sztyber-Betley", "Owain Evans" ], "aliases": [ "https://doi.org/10.48550/arXiv.2607.14345", "https://arxiv.org/abs/2607.14345", "https://arxiv.org/pdf/2607.14345" ] }, { "id": "agent-security-red-teaming-publication", "title": "Insights into AI Agent Security from a Large-Scale Red-Teaming Competition", "url": "https://www.nist.gov/blogs/caisi-research-blog/insights-ai-agent-security-large-scale-red-teaming-competition", "version": "Source capture 2026-10-25", "publicationDate": "2026-03-23", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.nist.gov/blogs/caisi-research-blog/insights-ai-agent-security-large-scale-red-teaming-competition" }, { "id": "labbench-two-publication", "title": "LABBench2: An Improved Benchmark for AI Systems Performing Biology Research", "url": "https://arxiv.org/abs/2604.09554", "version": "arXiv:2604.09554v2", "publicationDate": "2026-02-04", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2604.09554", "authors": [ "Jon M Laurent", "Albert Bou", "Michael Pieler", "Conor Igoe", "Alex Andonian", "Siddharth Narayanan", "James Braza", "Alexandros Sanchez Vassopoulos", "Jacob L Steenwyk", "Blake Lash", "Andrew D White", "Samuel G Rodriques" ], "aliases": [ "https://doi.org/10.48550/arXiv.2604.09554", "https://arxiv.org/abs/2604.09554", "https://arxiv.org/pdf/2604.09554" ] }, { "id": "labbench-two-publication-author-source", "title": "LABBench2 An Improved Benchmark for AI Systems Performing Biology Research", "url": "https://arxiv.org/html/2604.09554", "version": "arXiv:2604.09554v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2604.09554", "authors": [ "Jon M Laurent", "Albert Bou", "Michael Pieler", "Conor Igoe", "Alex Andonian", "Siddharth Narayanan", "James Braza", "Alexandros Sanchez Vassopoulos", "Jacob L Steenwyk", "Blake Lash", "Andrew D White", "Samuel G Rodriques" ], "aliases": [ "https://doi.org/10.48550/arXiv.2604.09554", "https://arxiv.org/abs/2604.09554", "https://arxiv.org/pdf/2604.09554" ] }, { "id": "fortress-publication", "title": "FORTRESS: Frontier Risk Evaluation for National Security and Public Safety", "url": "https://arxiv.org/abs/2506.14922", "version": "arXiv:2506.14922v2", "publicationDate": "2025-06-17", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2506.14922", "authors": [ "Christina Q. Knight ∗", "Kaustubh Deshpande ⋄", "Ved Sirdeshmukh ⋄", "Meher Mankikar", "Scale Red Team", "SEAL Research Team", "and Julian Michael" ], "aliases": [ "https://doi.org/10.48550/arXiv.2506.14922", "https://arxiv.org/abs/2506.14922", "https://arxiv.org/pdf/2506.14922" ] }, { "id": "fortress-publication-author-source", "title": "FORTRESS: Frontier Risk Evaluation for National Security and Public Safety", "url": "https://arxiv.org/html/2506.14922", "version": "arXiv:2506.14922v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2506.14922", "authors": [ "Christina Q. Knight ∗", "Kaustubh Deshpande ⋄", "Ved Sirdeshmukh ⋄", "Meher Mankikar", "Scale Red Team", "SEAL Research Team", "and Julian Michael" ], "aliases": [ "https://doi.org/10.48550/arXiv.2506.14922", "https://arxiv.org/abs/2506.14922", "https://arxiv.org/pdf/2506.14922" ] }, { "id": "rand-biological-benchmarking-publication", "title": "Toward Comprehensive Benchmarking of the Biological Knowledge of Frontier Large Language Models", "url": "https://www.rand.org/pubs/research_reports/RRA3797-1.html", "version": "Source capture 2026-10-25", "publicationDate": "2025-11-25", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.rand.org/pubs/research_reports/RRA3797-1.html", "authors": [ "Dev, Sunishchal", "Teague, Charles", "Ellison, Grant", "Brady, Kyle", "Lee, Jeffrey", "Gebauer, Sarah L.", "Bradley, Henry Alexander", "Maciorowski, Dawid", "Persaud, Bria", "Despanie, Jordan", "Del Castello, Barbara", "Worland, Alyssa", "Miller, Michael", "Salas, Adrian", "Nguyen, Dave", "Liu, James", "Johnson, Jason", "Sloan, Andrew", "Stonehouse, Will", "Merrill, Travis", "Goode, Thomas", "McKelvey, Greg, Jr.", "Guest, Ella" ] }, { "id": "rand-biological-risk-publication", "title": "Contemporary Foundation AI Models Increase Biological Weapons Risk", "url": "https://www.rand.org/pubs/perspectives/PEA3853-1.html", "version": "Source capture 2026-10-25", "publicationDate": "2025-12-31", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.rand.org/pubs/perspectives/PEA3853-1.html", "authors": [ "Brent, Roger", "McKelvey, Greg, Jr." ] }, { "id": "conventional-weapons-evaluations-publication", "title": "Measuring tactical intelligence targeting and conventional weapons capabilities of AI models", "url": "https://www.anthropic.com/research/intelligence-targeting-conventional-weapons-capabilities", "version": "Source capture 2026-10-25", "publicationDate": "2026-09-10", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/research/intelligence-targeting-conventional-weapons-capabilities" }, { "id": "fortress-audit-publication", "title": "Auditing FORTRESS: A Multi-Judge Panel Doesn’t Guarantee Robustness", "url": "https://generality.org/blog/posts/fortress-fragile-judges/", "version": "Source capture 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://generality.org/blog/posts/fortress-fragile-judges", "authors": [ "Christina Q. Knight ∗", "Kaustubh Deshpande ⋄", "Ved Sirdeshmukh ⋄", "Meher Mankikar", "Scale Red Team", "SEAL Research Team", "and Julian Michael" ] }, { "id": "fortress-audit-publication-author-source", "title": "FORTRESS: Frontier Risk Evaluation for National Security and Public Safety", "url": "https://arxiv.org/html/2506.14922", "version": "arXiv:2506.14922v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2506.14922", "authors": [ "Christina Q. Knight ∗", "Kaustubh Deshpande ⋄", "Ved Sirdeshmukh ⋄", "Meher Mankikar", "Scale Red Team", "SEAL Research Team", "and Julian Michael" ], "aliases": [ "https://doi.org/10.48550/arXiv.2506.14922", "https://arxiv.org/abs/2506.14922", "https://arxiv.org/pdf/2506.14922" ] }, { "id": "cheating-agent-evaluations-publication", "title": "Cheating On AI Agent Evaluations", "url": "https://www.nist.gov/blogs/caisi-research-blog/cheating-ai-agent-evaluations", "version": "Source capture 2026-10-25", "publicationDate": "2025-12-02", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.nist.gov/blogs/caisi-research-blog/cheating-ai-agent-evaluations" }, { "id": "item-response-safety-publication", "title": "Item Response Theory for AI Safety", "url": "https://www.aisi.gov.uk/research/item-response-theory-for-ai-safety", "version": "Source capture 2026-10-25", "publicationDate": "2026-08-05", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/item-response-theory-for-ai-safety", "authors": [ "Joshua Fonseca Rivera", "Neil Shah", "David Demitri Africa", "Konstantinos Voudouris" ] }, { "id": "item-response-safety-publication-author-source", "title": "Item Response Theory for AI Safety", "url": "https://arxiv.org/html/2608.05086", "version": "arXiv:2608.05086 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2608.05086", "authors": [ "Joshua Fonseca Rivera", "Neil Shah", "David Demitri Africa", "Konstantinos Voudouris" ], "aliases": [ "https://doi.org/10.48550/arXiv.2608.05086", "https://arxiv.org/abs/2608.05086", "https://arxiv.org/pdf/2608.05086" ] }, { "id": "optstop-publication", "title": "Knowing When to Stop: Bayesian Optimal Stopping for LLM Evaluations", "url": "https://www.aisi.gov.uk/research/knowing-when-to-stop-bayesian-optimal-stopping-for-llm-evaluations", "version": "Source capture 2026-10-25", "publicationDate": "2026-08-14", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/knowing-when-to-stop-bayesian-optimal-stopping-for-llm-evaluations", "authors": [ "Toby D. Pilditch" ] }, { "id": "optstop-publication-author-source", "title": "Knowing When to Stop: Bayesian Optimal Stopping for LLM Evaluations", "url": "https://arxiv.org/html/2608.14425", "version": "arXiv:2608.14425 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2608.14425", "authors": [ "Toby D. Pilditch" ], "aliases": [ "https://doi.org/10.48550/arXiv.2608.14425", "https://arxiv.org/abs/2608.14425", "https://arxiv.org/pdf/2608.14425" ] }, { "id": "metr-frontier-risk-report-publication", "title": "Frontier Risk Report (February to March 2026)", "url": "https://metr.org/blog/2026-05-19-frontier-risk-report/", "version": "Source capture 2026-10-25", "publicationDate": "2026-05-19", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://metr.org/blog/2026-05-19-frontier-risk-report", "authors": [ "Jonah Brown-Cohen", "David Lindner", "Rohin Shah" ] }, { "id": "metr-frontier-risk-report-publication-author-source", "title": "Quantifying the Necessity of Chain of Thought through Opaque Serial Depth", "url": "https://arxiv.org/html/2603.09786", "version": "arXiv:2603.09786 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2603.09786", "authors": [ "Jonah Brown-Cohen", "David Lindner", "Rohin Shah" ], "aliases": [ "https://doi.org/10.48550/arXiv.2603.09786", "https://arxiv.org/abs/2603.09786", "https://arxiv.org/pdf/2603.09786" ] }, { "id": "hugging-face-incident-publication", "title": "Brief independent investigation of agents’ behavior, reasoning and collaboration in the OpenAI / Hugging Face hacking incident", "url": "https://metr.org/blog/2026-08-26-openai-hugging-face-incident-investigation/", "version": "Source capture 2026-10-25", "publicationDate": "2026-08-26", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://metr.org/blog/2026-08-26-openai-hugging-face-incident-investigation" }, { "id": "anthropic-cyber-incidents-publication", "title": "An alignment assessment of recent cybersecurity incidents", "url": "https://www.anthropic.com/research/alignment-assessment-cybersecurity-incidents", "version": "Source capture 2026-10-25", "publicationDate": "2026-09-09", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/research/alignment-assessment-cybersecurity-incidents" }, { "id": "anthropic-threat-report-publication", "title": "Detecting and countering misuse of AI: September 2026", "url": "https://www.anthropic.com/threat-intelligence-report-september-2026", "version": "Source capture 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/threat-intelligence-report-september-2026" }, { "id": "saferai-glm-risk-publication", "title": "GLM-5.2 Risk Evaluation Report", "url": "https://www.safer-ai.org/research/glm-5-2-evaluation-report", "version": "Source capture 2026-10-25", "publicationDate": "2026-08-02", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.safer-ai.org/research/glm-5-2-evaluation-report" }, { "id": "bountybench-publication", "title": "BountyBench: Dollar Impact of AI Agent Attackers and Defenders on Real-World Cybersecurity Systems", "url": "https://arxiv.org/abs/2505.15216", "version": "arXiv:2505.15216v3", "publicationDate": "2025-05-21", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2505.15216", "authors": [ "Andy K. Zhang", "Joey Ji", "Celeste Menders", "Riya Dulepet", "Thomas Qin", "Ron Y. Wang", "Junrong Wu", "Kyleen Liao", "Jiliang Li", "Jinghan Hu", "Sara Hong", "Nardos Demilew", "Shivatmica Murgai", "Jason Tran", "Nishka Kacheria", "Ethan Ho", "Denis Liu", "Lauren McLane", "Olivia Bruvik", "Dai-Rong Han", "Seungwoo Kim", "Akhil Vyas", "Cuiyuanxiu Chen", "Ryan Li", "Weiran Xu", "Jonathan Z. Ye", "Prerit Choudhary", "Siddharth M. Bhatia", "Vikram Sivashankar", "Yuxuan Bao", "Dawn Song", "Dan Boneh", "Daniel E. Ho", "Percy Liang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2505.15216", "https://arxiv.org/abs/2505.15216", "https://arxiv.org/pdf/2505.15216" ] }, { "id": "bountybench-publication-author-source", "title": "BountyBench: Dollar Impact of AI Agent Attackers and Defenders on Real-World Cybersecurity Systems", "url": "https://arxiv.org/html/2505.15216", "version": "arXiv:2505.15216v3 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2505.15216", "authors": [ "Andy K. Zhang", "Joey Ji", "Celeste Menders", "Riya Dulepet", "Thomas Qin", "Ron Y. Wang", "Junrong Wu", "Kyleen Liao", "Jiliang Li", "Jinghan Hu", "Sara Hong", "Nardos Demilew", "Shivatmica Murgai", "Jason Tran", "Nishka Kacheria", "Ethan Ho", "Denis Liu", "Lauren McLane", "Olivia Bruvik", "Dai-Rong Han", "Seungwoo Kim", "Akhil Vyas", "Cuiyuanxiu Chen", "Ryan Li", "Weiran Xu", "Jonathan Z. Ye", "Prerit Choudhary", "Siddharth M. Bhatia", "Vikram Sivashankar", "Yuxuan Bao", "Dawn Song", "Dan Boneh", "Daniel E. Ho", "Percy Liang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2505.15216", "https://arxiv.org/abs/2505.15216", "https://arxiv.org/pdf/2505.15216" ] }, { "id": "deep-ignorance-publication", "title": "Deep Ignorance: Filtering Pretraining Data Builds Tamper-Resistant Safeguards into Open-Weight LLMs", "url": "https://arxiv.org/abs/2508.06601", "version": "arXiv:2508.06601v2", "publicationDate": "2025-08-08", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2508.06601", "authors": [ "Kyle O’Brien", "Stephen Casper", "Quentin Anthony", "Tomek Korbak", "Robert Kirk", "Xander Davies", "Ishan Mishra", "Geoffrey Irving", "Yarin Gal", "Stella Biderman" ], "aliases": [ "https://doi.org/10.48550/arXiv.2508.06601", "https://arxiv.org/abs/2508.06601", "https://arxiv.org/pdf/2508.06601" ] }, { "id": "deep-ignorance-publication-author-source", "title": "Deep Ignorance: Filtering Pretraining Data Builds Tamper-Resistant Safeguards into Open-Weight LLMs", "url": "https://arxiv.org/html/2508.06601", "version": "arXiv:2508.06601 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2508.06601", "authors": [ "Kyle O’Brien", "Stephen Casper", "Quentin Anthony", "Tomek Korbak", "Robert Kirk", "Xander Davies", "Ishan Mishra", "Geoffrey Irving", "Yarin Gal", "Stella Biderman" ], "aliases": [ "https://doi.org/10.48550/arXiv.2508.06601", "https://arxiv.org/abs/2508.06601", "https://arxiv.org/pdf/2508.06601" ] }, { "id": "monitoring-monitorability-publication", "title": "Monitoring Monitorability", "url": "https://arxiv.org/abs/2512.18311", "version": "arXiv:2512.18311v1", "publicationDate": "2025-12-20", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2512.18311", "authors": [ "Melody Y. Guan 2 2 2 † \\dagger Equal contribution. Correspondence to Bowen Baker at bowen@openai.com", "Miles Wang 2 2 footnotemark: 2", "Micah Carroll 2 2 footnotemark: 2", "Annie Y. Wei", "Marcus Williams", "Benjamin Arnav", "Ian Kivlichan", "Mia Glaese", "Jakub Pachocki" ], "aliases": [ "https://doi.org/10.48550/arXiv.2512.18311", "https://arxiv.org/abs/2512.18311", "https://arxiv.org/pdf/2512.18311" ] }, { "id": "monitoring-monitorability-publication-author-source", "title": "Monitoring Monitorability", "url": "https://arxiv.org/html/2512.18311", "version": "arXiv:2512.18311 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2512.18311", "authors": [ "Melody Y. Guan 2 2 2 † \\dagger Equal contribution. Correspondence to Bowen Baker at bowen@openai.com", "Miles Wang 2 2 footnotemark: 2", "Micah Carroll 2 2 footnotemark: 2", "Annie Y. Wei", "Marcus Williams", "Benjamin Arnav", "Ian Kivlichan", "Mia Glaese", "Jakub Pachocki" ], "aliases": [ "https://doi.org/10.48550/arXiv.2512.18311", "https://arxiv.org/abs/2512.18311", "https://arxiv.org/pdf/2512.18311" ] }, { "id": "monitorability-opportunity-publication", "title": "Chain of Thought Monitorability: A New and Fragile Opportunity for AI Safety", "url": "https://arxiv.org/abs/2507.11473", "version": "arXiv:2507.11473v2", "publicationDate": "2025-07-15", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2507.11473", "authors": [ "Tomek Korbak * UK AI Security Institute Mikita Balesni * Apollo Research Elizabeth Barnes METR Yoshua Bengio University of Montreal & Mila Joe Benton Anthropic Joseph Bloom UK AI Security Institute Mark Chen OpenAI Alan Cooney UK AI Security Institute Allan Dafoe Google DeepMind Anca Dragan Google DeepMind Scott Emmons Google DeepMind Owain Evans Truthful AI & UC Berkeley David Farhi OpenAI Ryan Greenblatt Redwood Research Dan Hendrycks Center for AI Safety Marius Hobbhahn Apollo Research Evan Hubinger Anthropic Geoffrey Irving UK AI Security Institute Erik Jenner Google DeepMind Daniel Kokotajlo AI Futures Project Victoria Krakovna Google DeepMind Shane Legg Google DeepMind David Lindner Google DeepMind David Luan Amazon Aleksander Mądry OpenAI Julian Michael Scale AI Neel Nanda Google DeepMind Dave Orr Google DeepMind Jakub Pachocki OpenAI Ethan Perez Anthropic Mary Phuong Google DeepMind Fabien Roger Anthropic Joshua Saxe Meta Buck Shlegeris Redwood Research Martín Soto UK AI Security Institute Eric Steinberger Magic Jasmine Wang UK AI Security Institute Wojciech Zaremba OpenAI Bowen Baker † OpenAI Rohin Shah † Google DeepMind Vlad Mikulik † Anthropic" ], "aliases": [ "https://doi.org/10.48550/arXiv.2507.11473", "https://arxiv.org/abs/2507.11473", "https://arxiv.org/pdf/2507.11473" ] }, { "id": "monitorability-opportunity-publication-author-source", "title": "Chain of Thought Monitorability: A New and Fragile Opportunity for AI Safety", "url": "https://arxiv.org/html/2507.11473", "version": "arXiv:2507.11473 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2507.11473", "authors": [ "Tomek Korbak * UK AI Security Institute Mikita Balesni * Apollo Research Elizabeth Barnes METR Yoshua Bengio University of Montreal & Mila Joe Benton Anthropic Joseph Bloom UK AI Security Institute Mark Chen OpenAI Alan Cooney UK AI Security Institute Allan Dafoe Google DeepMind Anca Dragan Google DeepMind Scott Emmons Google DeepMind Owain Evans Truthful AI & UC Berkeley David Farhi OpenAI Ryan Greenblatt Redwood Research Dan Hendrycks Center for AI Safety Marius Hobbhahn Apollo Research Evan Hubinger Anthropic Geoffrey Irving UK AI Security Institute Erik Jenner Google DeepMind Daniel Kokotajlo AI Futures Project Victoria Krakovna Google DeepMind Shane Legg Google DeepMind David Lindner Google DeepMind David Luan Amazon Aleksander Mądry OpenAI Julian Michael Scale AI Neel Nanda Google DeepMind Dave Orr Google DeepMind Jakub Pachocki OpenAI Ethan Perez Anthropic Mary Phuong Google DeepMind Fabien Roger Anthropic Joshua Saxe Meta Buck Shlegeris Redwood Research Martín Soto UK AI Security Institute Eric Steinberger Magic Jasmine Wang UK AI Security Institute Wojciech Zaremba OpenAI Bowen Baker † OpenAI Rohin Shah † Google DeepMind Vlad Mikulik † Anthropic" ], "aliases": [ "https://doi.org/10.48550/arXiv.2507.11473", "https://arxiv.org/abs/2507.11473", "https://arxiv.org/pdf/2507.11473" ] }, { "id": "monitorbench-publication", "title": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "url": "https://arxiv.org/abs/2603.28590v2", "version": "arXiv:2603.28590v2", "publicationDate": "2026-03-30", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2603.28590", "authors": [ "Han Wang", "Yifan Sun", "Brian Ko", "Mann Talati", "Jiawen Gong", "Zimeng Li   Naicheng Yu   Xucheng Yu   Wei Shen   Vedant Jolly", "Huan Zhang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2603.28590", "https://arxiv.org/abs/2603.28590", "https://arxiv.org/pdf/2603.28590" ] }, { "id": "monitorbench-publication-author-source", "title": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "url": "https://arxiv.org/html/2603.28590v2", "version": "arXiv:2603.28590v2", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2603.28590", "authors": [ "Han Wang", "Yifan Sun", "Brian Ko", "Mann Talati", "Jiawen Gong", "Zimeng Li   Naicheng Yu   Xucheng Yu   Wei Shen   Vedant Jolly", "Huan Zhang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2603.28590", "https://arxiv.org/abs/2603.28590", "https://arxiv.org/pdf/2603.28590" ] }, { "id": "catastrophic-cyber-benchmark-publication", "title": "Catastrophic Cyber Capabilities Benchmark (3CB): Robustly Evaluating LLM Agent Cyber Offense Capabilities", "url": "https://arxiv.org/abs/2410.09114", "version": "arXiv:2410.09114v2", "publicationDate": "2024-10-10", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2410.09114", "authors": [ "Your Name" ], "aliases": [ "https://doi.org/10.48550/arXiv.2410.09114", "https://arxiv.org/abs/2410.09114", "https://arxiv.org/pdf/2410.09114" ] }, { "id": "catastrophic-cyber-benchmark-publication-author-source", "title": "Catastrophic Cyber Capabilities Benchmark (3CB): Robustly Evaluating LLM Agent Cyber Offense Capabilities", "url": "https://arxiv.org/html/2410.09114", "version": "arXiv:2410.09114v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2410.09114", "authors": [ "Your Name" ], "aliases": [ "https://doi.org/10.48550/arXiv.2410.09114", "https://arxiv.org/abs/2410.09114", "https://arxiv.org/pdf/2410.09114" ] }, { "id": "three-cb-attribution", "title": "‘3cb’: The Catastrophic Cyber Capabilities Benchmark", "url": "https://apartresearch.com/news/3cb-the-catastrophic-cyber-capabilities-benchmark", "version": "Source capture 2026-10-25", "publicationDate": "2024-10-30", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://apartresearch.com/news/3cb-the-catastrophic-cyber-capabilities-benchmark" }, { "id": "instrumentaleval-publication", "title": "Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "url": "https://arxiv.org/abs/2502.12206", "version": "arXiv:2502.12206v1", "publicationDate": "2025-02-16", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2502.12206", "authors": [ "Yufei He", "Yuexin Li", "Jiaying Wu", "Yuan Sui", "Yulin Chen", "Bryan Hooi" ], "aliases": [ "https://doi.org/10.48550/arXiv.2502.12206", "https://arxiv.org/abs/2502.12206", "https://arxiv.org/pdf/2502.12206" ] }, { "id": "instrumentaleval-publication-author-source", "title": "Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "url": "https://arxiv.org/html/2502.12206", "version": "arXiv:2502.12206v1 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2502.12206", "authors": [ "Yufei He", "Yuexin Li", "Jiaying Wu", "Yuan Sui", "Yulin Chen", "Bryan Hooi" ], "aliases": [ "https://doi.org/10.48550/arXiv.2502.12206", "https://arxiv.org/abs/2502.12206", "https://arxiv.org/pdf/2502.12206" ] }, { "id": "mcptox-publication", "title": "MCPTox: A Benchmark for Tool Poisoning Attack on Real-World MCP Servers", "url": "https://arxiv.org/abs/2508.14925", "version": "arXiv:2508.14925v2", "publicationDate": "2025-08-19", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2508.14925", "authors": [ "Zhiqiang Wang", "Yichao Gao", "Yanting Wang", "Suyuan Liu", "Haifeng Sun", "Haoran Cheng", "Guanquan Shi", "Haohua Du", "Xiang-Yang Li" ], "aliases": [ "https://doi.org/10.48550/arXiv.2508.14925", "https://arxiv.org/abs/2508.14925", "https://arxiv.org/pdf/2508.14925" ] }, { "id": "mcptox-publication-author-source", "title": "MCPTox: A Benchmark for Tool Poisoning Attack on Real-World MCP Servers", "url": "https://arxiv.org/html/2508.14925", "version": "arXiv:2508.14925v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2508.14925", "aliases": [ "https://doi.org/10.48550/arXiv.2508.14925", "https://arxiv.org/abs/2508.14925", "https://arxiv.org/pdf/2508.14925" ] }, { "id": "judgebench-publication", "title": "JudgeBench: A Benchmark for Evaluating LLM-based Judges", "url": "https://arxiv.org/abs/2410.12784v2", "version": "arXiv:2410.12784v2", "publicationDate": "2024-10-16", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2410.12784", "authors": [ "Sijun Tan", "Siyuan Zhuang", "Kyle Montgomery", "William Y. Tang", "Alejandro Cuadron", "Chenguang Wang", "Raluca Ada Popa", "Ion Stoica" ], "aliases": [ "https://doi.org/10.48550/arXiv.2410.12784", "https://arxiv.org/abs/2410.12784", "https://arxiv.org/pdf/2410.12784" ] }, { "id": "judgebench-publication-author-source", "title": "JudgeBench: A Benchmark for Evaluating LLM-Based Judges", "url": "https://arxiv.org/html/2410.12784v2", "version": "arXiv:2410.12784v2", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2410.12784", "authors": [ "Sijun Tan", "Siyuan Zhuang", "Kyle Montgomery", "William Y. Tang", "Alejandro Cuadron", "Chenguang Wang", "Raluca Ada Popa", "Ion Stoica" ], "aliases": [ "https://doi.org/10.48550/arXiv.2410.12784", "https://arxiv.org/abs/2410.12784", "https://arxiv.org/pdf/2410.12784" ] }, { "id": "narcbench-publication", "title": "Detecting Multi-Agent Collusion Through Multi-Agent Interpretability", "url": "https://arxiv.org/abs/2604.01151v2", "version": "arXiv:2604.01151v2", "publicationDate": "2026-04-01", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2604.01151", "authors": [ "Aaron Rose", "Carissa Cullen", "Sahar Abdelnabi", "Philip Torr", "Brandon Gary Kaplowitz", "Christian Schroeder de Witt" ], "aliases": [ "https://doi.org/10.48550/arXiv.2604.01151", "https://arxiv.org/abs/2604.01151", "https://arxiv.org/pdf/2604.01151" ] }, { "id": "narcbench-publication-author-source", "title": "Detecting Multi-Agent Collusion Through Multi-Agent Interpretability", "url": "https://arxiv.org/html/2604.01151v2", "version": "arXiv:2604.01151v2", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2604.01151", "authors": [ "Aaron Rose", "Carissa Cullen", "Sahar Abdelnabi", "Philip Torr", "Brandon Gary Kaplowitz", "Christian Schroeder de Witt" ], "aliases": [ "https://doi.org/10.48550/arXiv.2604.01151", "https://arxiv.org/abs/2604.01151", "https://arxiv.org/pdf/2604.01151" ] }, { "id": "transect-publication", "title": "Transect: Retaining Observability for Long-Horizon LLM Agent Evaluations", "url": "https://www.aisi.gov.uk/research/transect-retaining-observability-for-long-horizon-llm-agent-evaluations", "version": "Source capture 2026-10-25", "publicationDate": "2026-10-07", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/transect-retaining-observability-for-long-horizon-llm-agent-evaluations" }, { "id": "extreme-risk-evaluations-publication", "title": "Model evaluation for extreme risks", "url": "https://arxiv.org/abs/2305.15324", "version": "arXiv:2305.15324v2", "publicationDate": "2023-05-24", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2305.15324", "authors": [ "Toby Shevlane", "Sebastian Farquhar", "Ben Garfinkel", "Mary Phuong", "Jess Whittlestone", "Jade Leung", "Daniel Kokotajlo", "Nahema Marchal", "Markus Anderljung", "Noam Kolt", "Lewis Ho", "Divya Siddarth", "Shahar Avin", "Will Hawkins", "Been Kim", "Iason Gabriel", "Vijay Bolina", "Jack Clark", "Yoshua Bengio", "Paul Christiano", "Allan Dafoe" ], "aliases": [ "https://doi.org/10.48550/arXiv.2305.15324", "https://arxiv.org/abs/2305.15324", "https://arxiv.org/pdf/2305.15324" ] }, { "id": "extreme-risk-evaluations-publication-author-source", "title": "Model evaluation for extreme risks", "url": "https://arxiv.org/html/2305.15324", "version": "arXiv:2305.15324 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2305.15324", "authors": [ "Toby Shevlane", "Sebastian Farquhar", "Ben Garfinkel", "Mary Phuong", "Jess Whittlestone", "Jade Leung", "Daniel Kokotajlo", "Nahema Marchal", "Markus Anderljung", "Noam Kolt", "Lewis Ho", "Divya Siddarth", "Shahar Avin", "Will Hawkins", "Been Kim", "Iason Gabriel", "Vijay Bolina", "Jack Clark", "Yoshua Bengio", "Paul Christiano", "Allan Dafoe" ], "aliases": [ "https://doi.org/10.48550/arXiv.2305.15324", "https://arxiv.org/abs/2305.15324", "https://arxiv.org/pdf/2305.15324" ] }, { "id": "monitorability-metrics-publication", "title": "Beyond g-mean²: complementary metrics for monitorability", "url": "https://alignment.openai.com/beyond-gmean-squared/", "version": "Source capture 2026-10-25", "publicationDate": "2026-10-20", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://alignment.openai.com/beyond-gmean-squared", "authors": [ "Melody Guan", "Yuxiang Xie", "Bowen Baker" ] }, { "id": "prefill-awareness-publication", "title": "Prefill Awareness in Large Language Models", "url": "https://www.aisi.gov.uk/research/prefill-awareness-in-large-language-models", "version": "Source capture 2026-10-25", "publicationDate": "2026-06-10", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/prefill-awareness-in-large-language-models" }, { "id": "consistency-misalignment-publication", "title": "Consistency Training Can Entrench Misalignment", "url": "https://www.aisi.gov.uk/research/consistency-training-can-entrench-misalignment", "version": "Source capture 2026-10-25", "publicationDate": "2026-06-02", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/consistency-training-can-entrench-misalignment" }, { "id": "loss-of-oversight-publication", "title": "Loss of Oversight: How AI systems may become harder to audit, monitor, and investigate", "url": "https://www.aisi.gov.uk/research/loss-of-oversight-how-ai-systems-may-become-harder-to-audit-monitor-and-investigate", "version": "Source capture 2026-10-25", "publicationDate": "2026-05-21", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/loss-of-oversight-how-ai-systems-may-become-harder-to-audit-monitor-and-investigate" }, { "id": "steganography-monitoring-publication", "title": "A Decision-Theoretic Formalisation of Steganography With Applications to LLM Monitoring", "url": "https://www.aisi.gov.uk/research/a-decision-theoretic-formalisation-of-steganography-with-applications-to-llm-monitoring", "version": "Source capture 2026-10-25", "publicationDate": "2026-04-29", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/a-decision-theoretic-formalisation-of-steganography-with-applications-to-llm-monitoring" }, { "id": "log-analysis-pipeline-publication", "title": "Seven simple steps for log analysis in AI systems", "url": "https://www.aisi.gov.uk/research/seven-simple-steps-for-log-analysis-in-ai-systems", "version": "Source capture 2026-10-25", "publicationDate": "2026-02-25", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/seven-simple-steps-for-log-analysis-in-ai-systems" }, { "id": "control-monitoring-deployment-publication", "title": "Practical challenges of control monitoring in frontier AI deployments", "url": "https://www.aisi.gov.uk/research/practical-challenges-of-control-monitoring-in-frontier-ai-deployments", "version": "Source capture 2026-10-25", "publicationDate": "2025-12-15", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/practical-challenges-of-control-monitoring-in-frontier-ai-deployments" }, { "id": "async-control-publication", "title": "Async control: Stress-testing asynchronous control measures for LLM agents", "url": "https://www.aisi.gov.uk/research/async-control-stress-testing-asynchronous-control-measures-for-llm-agents", "version": "Source capture 2026-10-25", "publicationDate": "2025-12-15", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/async-control-stress-testing-asynchronous-control-measures-for-llm-agents" }, { "id": "self-evaluation-wireheading-publication", "title": "Does self-evaluation enable wireheading in language models?", "url": "https://www.aisi.gov.uk/research/does-self-evaluation-enable-wireheading-in-language-models", "version": "Source capture 2026-10-25", "publicationDate": "2025-12-01", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/does-self-evaluation-enable-wireheading-in-language-models" }, { "id": "open-weight-risk-management-publication", "title": "Open technical problems in open-weight AI model risk management", "url": "https://www.aisi.gov.uk/research/open-technical-problems-in-open-weight-ai-model-risk-management", "version": "Source capture 2026-10-25", "publicationDate": "2025-10-26", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/open-technical-problems-in-open-weight-ai-model-risk-management" }, { "id": "pretraining-poisoning-publication", "title": "Poisoning Attacks on LLMs Require a Near-constant Number of Poison Samples", "url": "https://www.aisi.gov.uk/research/poisoning-attacks-on-llms-require-a-near-constant-number-of-poison-samples", "version": "Source capture 2026-10-25", "publicationDate": "2025-10-08", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/poisoning-attacks-on-llms-require-a-near-constant-number-of-poison-samples", "authors": [ "Alexandra Souly", "Javier Rando", "Ed Chapman", "Xander Davies", "Burak Hasircioglu", "Ezzeldin Shereen", "Carlos Mougan", "Vasilios Mavroudis", "Erik Jones", "Chris Hicks", "Nicholas Carlini", "Yarin Gal", "Robert Kirk" ] }, { "id": "pretraining-poisoning-publication-author-source", "title": "Poisoning Attacks on LLMs Require a Near-constant Number of Poison Samples", "url": "https://arxiv.org/html/2510.07192", "version": "arXiv:2510.07192 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2510.07192", "authors": [ "Alexandra Souly", "Javier Rando", "Ed Chapman", "Xander Davies", "Burak Hasircioglu", "Ezzeldin Shereen", "Carlos Mougan", "Vasilios Mavroudis", "Erik Jones", "Chris Hicks", "Nicholas Carlini", "Yarin Gal", "Robert Kirk" ], "aliases": [ "https://doi.org/10.48550/arXiv.2510.07192", "https://arxiv.org/abs/2510.07192", "https://arxiv.org/pdf/2510.07192" ] }, { "id": "inoculation-prompting-publication", "title": "Inoculation Prompting: Eliciting traits from LLMs during training can suppress them at test-time", "url": "https://www.aisi.gov.uk/research/inoculation-prompting-eliciting-traits-from-llms-during-training-can-suppress-them-at-test-time", "version": "Source capture 2026-10-25", "publicationDate": "2025-10-05", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/inoculation-prompting-eliciting-traits-from-llms-during-training-can-suppress-them-at-test-time", "authors": [ "Daniel Tan", "Anders Woodruff 2 2 footnotemark: 2", "Niels Warncke 2 2 footnotemark: 2", "Arun Jose 2 2 footnotemark: 2", "Maxime Riché 2 2 footnotemark: 2", "David Demitri Africa", "Mia Taylor 2 2 footnotemark: 2" ] }, { "id": "inoculation-prompting-publication-author-source", "title": "Inoculation Prompting: Eliciting traits from LLMs during training can suppress them at test-time", "url": "https://arxiv.org/html/2510.04340", "version": "arXiv:2510.04340 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2510.04340", "authors": [ "Daniel Tan", "Anders Woodruff 2 2 footnotemark: 2", "Niels Warncke 2 2 footnotemark: 2", "Arun Jose 2 2 footnotemark: 2", "Maxime Riché 2 2 footnotemark: 2", "David Demitri Africa", "Mia Taylor 2 2 footnotemark: 2" ], "aliases": [ "https://doi.org/10.48550/arXiv.2510.04340", "https://arxiv.org/abs/2510.04340", "https://arxiv.org/pdf/2510.04340" ] }, { "id": "misuse-safety-case-publication", "title": "An example safety case for safeguards against misuse", "url": "https://www.aisi.gov.uk/research/an-example-safety-case-for-safeguards-against-misuse", "version": "Source capture 2026-10-25", "publicationDate": "2025-06-05", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/an-example-safety-case-for-safeguards-against-misuse" }, { "id": "unlearning-evaluation-validity-publication", "title": "Existing Large Language Model unlearning evaluations are inconclusive", "url": "https://www.aisi.gov.uk/research/existing-large-language-model-unlearning-evaluations-are-inconclusive", "version": "Source capture 2026-10-25", "publicationDate": "2025-05-31", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/existing-large-language-model-unlearning-evaluations-are-inconclusive" }, { "id": "safety-cases-publication", "title": "Safety Cases: A scalable approach to Frontier AI safety", "url": "https://www.aisi.gov.uk/research/safety-cases-a-scalable-approach-to-frontier-ai-safety", "version": "Source capture 2026-10-25", "publicationDate": "2025-02-10", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/safety-cases-a-scalable-approach-to-frontier-ai-safety" }, { "id": "misuse-safeguard-principles-publication", "title": "Principles for evaluating misuse safeguards of frontier AI systems", "url": "https://www.aisi.gov.uk/research/principles-for-evaluating-misuse-safeguards-of-frontier-ai-systems", "version": "Source capture 2026-10-25", "publicationDate": "2025-02-04", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/principles-for-evaluating-misuse-safeguards-of-frontier-ai-systems" }, { "id": "control-safety-case-publication", "title": "A sketch of an AI control safety case", "url": "https://www.aisi.gov.uk/research/a-sketch-of-an-ai-control-safety-case", "version": "Source capture 2026-10-25", "publicationDate": "2025-01-28", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.aisi.gov.uk/research/a-sketch-of-an-ai-control-safety-case", "authors": [ "Tomek Korbak tomek.korbak@dsit.gov.uk", "Joshua Clymer joshua@rdwrs.com", "Benjamin Hilton benjamin.hilton@dsit.gov.uk", "Buck Shlegeris buck@rdwrs.com", "Geoffrey Irving geoffrey.irving@dsit.gov.uk" ] }, { "id": "control-safety-case-publication-author-source", "title": "A sketch of an AI control safety case", "url": "https://arxiv.org/html/2501.17315", "version": "arXiv:2501.17315 (source capture 2026-10-25)", "publicationDate": null, "datePrecision": "unknown", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2501.17315", "authors": [ "Tomek Korbak tomek.korbak@dsit.gov.uk", "Joshua Clymer joshua@rdwrs.com", "Benjamin Hilton benjamin.hilton@dsit.gov.uk", "Buck Shlegeris buck@rdwrs.com", "Geoffrey Irving geoffrey.irving@dsit.gov.uk" ], "aliases": [ "https://doi.org/10.48550/arXiv.2501.17315", "https://arxiv.org/abs/2501.17315", "https://arxiv.org/pdf/2501.17315" ] }, { "id": "embedded-evaluators-publication", "title": "Embedded Evaluators are necessary for meaningful external testing", "url": "https://www.apolloresearch.ai/blog/embedded-evaluators-are-necessary-for-meaningful-external-testing", "version": "Source capture 2026-10-25", "publicationDate": "2026-09-29", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://www.apolloresearch.ai/blog/embedded-evaluators-are-necessary-for-meaningful-external-testing" }, { "id": "specification-gaming-publication", "title": "Demonstrating specification gaming in reasoning models", "url": "https://palisaderesearch.org/research/specification-gaming", "version": "Source capture 2026-10-25", "publicationDate": "2025-02-19", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://palisaderesearch.org/research/specification-gaming" }, { "id": "llm-honeypot-publication", "title": "LLM Honeypot: an early warning system for autonomous hacking", "url": "https://palisaderesearch.org/research/llm-honeypot", "version": "Source capture 2026-10-25", "publicationDate": "2024-10-17", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://palisaderesearch.org/research/llm-honeypot" }, { "id": "beyond-owls-publication", "title": "Beyond Owls: Subliminal Learning Can Transfer Learned Capabilities and Backdoors", "url": "https://truthful.ai/papers/beyond-owls/", "version": "Source capture 2026-10-25", "publicationDate": "2026-10-07", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://truthful.ai/papers/beyond-owls", "authors": [ "Jan Dubiński", "Anna Sztyber-Betley", "Jan Betley", "Owain Evans" ] }, { "id": "activation-oracles-publication", "title": "Activation Oracles: Training and Evaluating LLMs as General-Purpose Activation Explainers", "url": "https://truthful.ai/papers/activation-oracles/", "version": "Source capture 2026-10-25", "publicationDate": "2025-12-19", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://truthful.ai/papers/activation-oracles", "authors": [ "Adam Karvonen", "James Chua", "Clément Dumas", "Kit Fraser-Taliente", "Subhash Kantamneni", "Julian Minder", "Euan Ong", "Arnab Sen Sharma", "Daniel Wen", "Owain Evans", "Samuel Marks" ] }, { "id": "rogue-replication-threat-publication", "title": "The Rogue Replication Threat Model", "url": "https://metr.org/blog/2024-11-12-rogue-replication-threat-model/", "version": "Source capture 2026-10-25", "publicationDate": "2024-11-12", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://metr.org/blog/2024-11-12-rogue-replication-threat-model" }, { "id": "capability-elicitation-publication", "title": "Guidelines for capability elicitation", "url": "https://metr.org/blog/2024-03-15-guidelines-for-capability-elicitation/", "version": "Source capture 2026-10-25", "publicationDate": "2024-03-15", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "https://metr.org/blog/2024-03-15-guidelines-for-capability-elicitation" }, { "id": "mcptox-aaai-author-source", "title": "MCPTox: A Benchmark for Tool Poisoning Attack on Real-World MCP Servers — AAAI-26 paper", "url": "https://ojs.aaai.org/index.php/AAAI/article/download/40895/44856", "version": "AAAI-26 proceedings, pp. 35811–35819", "publicationDate": null, "datePrecision": "year", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "https://ojs.aaai.org/index.php/AAAI/article/download/40895/44856", "authors": [ "Zhiqiang Wang", "Yichao Gao", "Yanting Wang", "Suyuan Liu", "Haifeng Sun", "Haoran Cheng", "Guanquan Shi", "Haohua Du", "Xiang-Yang Li" ] }, { "id": "paperbench-chart-source", "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research", "url": "https://arxiv.org/html/2504.01848v3", "version": "arXiv:2504.01848v3", "publicationDate": "2025-04-07", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "arxiv:2504.01848", "aliases": [ "https://doi.org/10.48550/arXiv.2504.01848", "https://arxiv.org/abs/2504.01848", "https://arxiv.org/pdf/2504.01848" ] }, { "id": "asimov-figure-source", "title": "Can AI Perceive Physical Danger and Intervene?", "url": "https://arxiv.org/html/2509.21651", "version": "arXiv:2509.21651v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2509.21651", "authors": [ "Abhishek Jindal", "Dmitry Kalashnikov", "R. Alex Hofer", "Oscar Chang", "Divya Garikapati", "Anirudha Majumdar", "Pierre Sermanet", "Vikas Sindhwani" ], "aliases": [ "https://doi.org/10.48550/arXiv.2509.21651", "https://arxiv.org/abs/2509.21651", "https://arxiv.org/pdf/2509.21651" ] }, { "id": "cybergym-figure-source", "title": "CyberGym: Evaluating AI Agents' Real-World Cybersecurity Capabilities at Scale", "url": "https://arxiv.org/html/2506.02548v3", "version": "arXiv:2506.02548v3", "publicationDate": "2026-03-24", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2506.02548", "authors": [ "Zhun Wang", "Tianneng Shi 1 1 footnotemark: 1", "Jingxuan He", "Matthew Cai", "Jialin Zhang", "Dawn Song", "{ zhun.wang", "stneng }@berkeley.edu" ], "aliases": [ "https://doi.org/10.48550/arXiv.2506.02548", "https://arxiv.org/abs/2506.02548", "https://arxiv.org/pdf/2506.02548" ] }, { "id": "in-context-scheming-figure-source", "title": "Frontier Models are Capable of In-context Scheming", "url": "https://arxiv.org/html/2412.04984v2", "version": "arXiv:2412.04984v2", "publicationDate": "2025-01-14", "datePrecision": "day", "type": "controlled-study", "verifiedOn": "2026-10-24", "documentId": "arxiv:2412.04984", "authors": [ "Alexander Meinke", "Bronson Schoen 1 1 footnotemark: 1", "Jérémy Scheurer 1 1 footnotemark: 1", "Mikita Balesni", "Rusheb Shah", "Marius Hobbhahn", "Apollo Research" ], "aliases": [ "https://doi.org/10.48550/arXiv.2412.04984", "https://arxiv.org/abs/2412.04984", "https://arxiv.org/pdf/2412.04984" ] }, { "id": "aisi-cyber-scenarios-figure-source", "title": "Measuring AI Agents’ Progress on Multi-Step Cyber Attack Scenarios", "url": "https://arxiv.org/html/2603.11214", "version": "arXiv:2603.11214v3 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "research-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2603.11214", "authors": [ "Linus Folkerts", "Will Payne", "Simon Inman", "Philippos Giavridis", "Joe Skinner", "Sam Deverett †", "James Aung", "Ekin Zorer", "Michael Schmatz", "Mahmoud Ghanem", "John Wilkinson", "Alan Steer", "Vy Hong", "Jessica Wang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2603.11214", "https://arxiv.org/abs/2603.11214", "https://arxiv.org/pdf/2603.11214" ] }, { "id": "time-horizons-figure-source", "title": "Task-Completion Time Horizons of Frontier AI Models", "url": "https://arxiv.org/html/2503.14499", "version": "arXiv:2503.14499v4 (2026-10-25 capture)", "publicationDate": "2026-09-08", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2503.14499", "authors": [ "Thomas Kwa", "Ben West 1 1 footnotemark: 1", "Joel Becker", "Amy Deng", "Katharyn GarciaMax Hasin , Sami Jawhar , Megan Kinniment , Nate Rush , Sydney Von ArxRyan Bloom , Thomas Broadley , Haoxing Du , Brian Goodrich , Nikola Jurkovic , Luke Harold Miles , Seraphina Nix , Tao Lin , Chris Painter , Neev Parikh , David Rein , Lucas Jun Koba Sato , Hjalmar Wijk , Daniel M. ZieglerElizabeth Barnes , Lawrence Chan Model Evaluation & Threat Research (METR)" ], "aliases": [ "https://doi.org/10.48550/arXiv.2503.14499", "https://arxiv.org/abs/2503.14499", "https://arxiv.org/pdf/2503.14499" ] }, { "id": "re-bench-figure-source", "title": "RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts", "url": "https://arxiv.org/html/2411.15114v2", "version": "arXiv:2411.15114v2", "publicationDate": "2025-05-27", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2411.15114", "authors": [ "Hjalmar Wijk", "Tao Lin", "Joel Becker", "Sami Jawhar", "Neev Parikh", "Thomas Broadley", "Lawrence Chan 2 2 footnotemark: 2", "Michael Chen 2 2 footnotemark: 2", "Josh Clymer 2 2 footnotemark: 2", "Jai Dhyani 2 2 footnotemark: 2", "Elena Ericheva 2 2 footnotemark: 2", "Katharyn Garcia 2 2 footnotemark: 2", "Brian Goodrich 2 2 footnotemark: 2", "Nikola Jurkovic 2 2 footnotemark: 2", "Holden Karnofsky 2 2 footnotemark: 2", "Megan Kinniment 2 2 footnotemark: 2", "Aron Lajko 2 2 footnotemark: 2 3 3 footnotemark: 3", "Seraphina Nix 2 2 footnotemark: 2", "Lucas Sato 2 2 footnotemark: 2", "William Saunders 2 2 footnotemark: 2", "Maksym Taran 2 2 footnotemark: 2", "Ben West 2 2 footnotemark: 2", "Elizabeth Barnes", "Model Evaluation and Threat Research (METR)" ], "aliases": [ "https://doi.org/10.48550/arXiv.2411.15114", "https://arxiv.org/abs/2411.15114", "https://arxiv.org/pdf/2411.15114" ] }, { "id": "truthful-sad-figure-source", "title": "Me, Myself, and AI: The Situational Awareness Dataset for LLMs", "url": "https://arxiv.org/html/2407.04694", "version": "arXiv:2407.04694v1 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "arxiv:2407.04694", "authors": [ "Rudolf Laine", "Bilal Chughtai", "Jan Betley", "Kaivalya Hariharan", "Jérémy Scheurer", "Mikita Balesni", "Marius Hobbhahn", "Alexander Meinke", "Owain Evans" ], "aliases": [ "https://doi.org/10.48550/arXiv.2407.04694", "https://arxiv.org/abs/2407.04694", "https://arxiv.org/pdf/2407.04694" ] }, { "id": "aisi-propensity-study-figure-source", "title": "Propensity Inference: Environmental Contributors to LLM Behaviour", "url": "https://arxiv.org/html/2604.21098", "version": "arXiv:2604.21098v1 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "research-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2604.21098", "authors": [ "Olli Järviniemi", "Oliver Makins", "Jacob Merizian", "Robert Kirk", "Ben Millwood" ], "aliases": [ "https://doi.org/10.48550/arXiv.2604.21098", "https://arxiv.org/abs/2604.21098", "https://arxiv.org/pdf/2604.21098" ] }, { "id": "palisade-shutdown-figure-source", "title": "Shutdown resistance in reasoning models", "url": "https://arxiv.org/html/2509.14260", "version": "arXiv:2509.14260v2 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "research-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2509.14260", "authors": [ "Jeremy Schlatter", "Benjamin Weinstein-Raun", "Jeffrey Ladish" ], "aliases": [ "https://doi.org/10.48550/arXiv.2509.14260", "https://arxiv.org/abs/2509.14260", "https://arxiv.org/pdf/2509.14260" ] }, { "id": "harmbench-figure-source", "title": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "url": "https://arxiv.org/html/2402.04249v2", "version": "arXiv:2402.04249v2", "publicationDate": "2024-02-27", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2402.04249", "authors": [ "Mantas Mazeika", "Long Phan", "Xuwang Yin", "Andy Zou", "Zifan Wang", "Norman Mu", "Elham Sakhaee", "Nathaniel Li", "Steven Basart", "Bo Li", "David Forsyth", "Dan Hendrycks" ], "aliases": [ "https://doi.org/10.48550/arXiv.2402.04249", "https://arxiv.org/abs/2402.04249", "https://arxiv.org/pdf/2402.04249" ] }, { "id": "political-persuasion-figure-source", "title": "Benchmarking Political Persuasion Risks Across Frontier Large Language Models", "url": "https://arxiv.org/html/2603.09884v1", "version": "arXiv:2603.09884v1", "publicationDate": "2026-03-10", "datePrecision": "day", "type": "controlled-study", "verifiedOn": "2026-10-24", "documentId": "arxiv:2603.09884", "authors": [ "Zhongren Chen", "Joshua Kalla", "Quan Le" ], "aliases": [ "https://doi.org/10.48550/arXiv.2603.09884", "https://arxiv.org/abs/2603.09884", "https://arxiv.org/pdf/2603.09884" ] }, { "id": "secbench-pro-figure-source", "title": "SEC-bench Pro: Can Language Models Solve Long-Horizon Software Security Tasks?", "url": "https://arxiv.org/html/2605.26548v2", "version": "arXiv:2605.26548v2", "publicationDate": "2026-07-20", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "arxiv:2605.26548", "authors": [ "Hwiwon Lee ∗ Jiawei Liu \nDongjun Kim \nWubing Xia \nZiqi Zhang \nChunqiu Steven Xia \nLingming Zhang ∗" ], "aliases": [ "https://doi.org/10.48550/arXiv.2605.26548", "https://arxiv.org/abs/2605.26548", "https://arxiv.org/pdf/2605.26548" ] }, { "id": "sandboxescapebench-figure-source", "title": "Quantifying Frontier LLM Capabilities for Container Sandbox Escape", "url": "https://arxiv.org/html/2603.02277", "version": "arXiv:2603.02277v3 (2026-10-25 capture)", "publicationDate": null, "datePrecision": "not recorded in catalogue", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "arxiv:2603.02277", "authors": [ "Rahul Marchand", "Art O Cathain", "Jerome Wynne", "Philippos Maximos Giavridis", "Stuart Jennings", "Freddy Tuxworth", "Tolga H. Dur", "Sam Deverett", "John Wilkinson", "Jason Gwartz ∗", "Harry Coppock ∗" ], "aliases": [ "https://doi.org/10.48550/arXiv.2603.02277", "https://arxiv.org/abs/2603.02277", "https://arxiv.org/pdf/2603.02277" ] }, { "id": "alignment-faking-figure-source", "title": "Alignment faking in large language models", "url": "https://arxiv.org/html/2412.14093", "version": "arXiv:2412.14093v2 (2026-10-25 capture)", "publicationDate": "2024-12-18", "datePrecision": "day", "type": "controlled-study", "verifiedOn": "2026-10-25", "documentId": "arxiv:2412.14093", "authors": [ "Ryan Greenblatt", "† Carson Denison † † footnotemark:", "Benjamin Wright † † footnotemark:", "Fabien Roger † † footnotemark:", "Monte MacDiarmid † † footnotemark:" ], "aliases": [ "https://doi.org/10.48550/arXiv.2412.14093", "https://arxiv.org/abs/2412.14093", "https://arxiv.org/pdf/2412.14093" ] }, { "id": "shade-arena-figure-source", "title": "SHADE-Arena: Evaluating sabotage and monitoring in LLM agents", "url": "https://www.arxiv.org/html/2506.15740", "version": "arXiv:2506.15740v2 (2026-10-25 capture)", "publicationDate": "2025-06-16", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "arxiv:2506.15740", "aliases": [ "https://doi.org/10.48550/arXiv.2506.15740", "https://arxiv.org/abs/2506.15740", "https://arxiv.org/pdf/2506.15740" ] }, { "id": "biotier-figure-source", "title": "BioTIER: A Refusal Benchmark for Targeted Biological Risk Mitigation", "url": "https://arxiv.org/html/2607.14479v1", "version": "arXiv:2607.14479v1", "publicationDate": "2026-07-16", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "arxiv:2607.14479", "authors": [ "Eleanor M. Marshall", "Pedro Medeiros", "Peter Peneder", "Nelly Mak", "Jacob Kaffey", "Faith Rovenolt", "Mac Walker", "Seth Donoughe", "Jasper Götting" ], "aliases": [ "https://doi.org/10.48550/arXiv.2607.14479", "https://arxiv.org/abs/2607.14479", "https://arxiv.org/pdf/2607.14479" ] }, { "id": "ape-figure-source", "title": "It's the Thought that Counts: Evaluating the Attempts of Frontier LLMs to Persuade on Harmful Topics", "url": "https://arxiv.org/html/2506.02873v4", "version": "arXiv:2506.02873v4", "publicationDate": "2026-02-15", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-25", "documentId": "arxiv:2506.02873", "authors": [ "\\authorname Matthew Kowal \\authorinstitution FAR.AI \\authorname Jasper Timm \\authorinstitution FAR.AI \\authorname Jean-Francois Godbout \\authorinstitution Université de Montréal, MILA \\authorname Thomas Costello \\authorinstitution Carnegie Mellon University \\authorname Antonio A. Arechar \\authorinstitution MIT, Center for Research and Teaching in Economics \\authorname Gordon Pennycook \\authorinstitution Cornell University, University of Regina \\authorname David Rand \\authorinstitution Cornell University, MIT \\authorname Adam Gleave \\authorinstitution FAR.AI \\authorname Kellin Pelrine \\authorinstitution FAR.AI" ], "aliases": [ "https://doi.org/10.48550/arXiv.2506.02873", "https://arxiv.org/abs/2506.02873", "https://arxiv.org/pdf/2506.02873" ] }, { "id": "sec-bench-figure-source", "title": "SEC-bench: Automated Benchmarking of LLM Agents on Real-World Software Security Tasks", "url": "https://arxiv.org/html/2506.11791", "version": "arXiv:2506.11791v2 (2026-10-25 capture)", "publicationDate": "2025-06-13", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2506.11791", "authors": [ "Hwiwon Lee \nZiqi Zhang \nHanxiao Lu † Lingming Zhang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2506.11791", "https://arxiv.org/abs/2506.11791", "https://arxiv.org/pdf/2506.11791" ] }, { "id": "exploitbench-figure-source", "title": "ExploitBench: A Capability Ladder Benchmark for LLM Cybersecurity Agents", "url": "https://arxiv.org/html/2605.14153", "version": "arXiv:2605.14153v1 (2026-10-25 capture)", "publicationDate": "2026-05-13", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2605.14153", "authors": [ "Seunghyun Lee", "David Brumley" ], "aliases": [ "https://doi.org/10.48550/arXiv.2605.14153", "https://arxiv.org/abs/2605.14153", "https://arxiv.org/pdf/2605.14153" ] }, { "id": "exploitgym-figure-source", "title": "ExploitGym: Can AI Agents Turn Security Vulnerabilities into Real Attacks?", "url": "https://arxiv.org/html/2605.11086", "version": "arXiv:2605.11086v1 (2026-10-25 capture)", "publicationDate": "2026-05-13", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "arxiv:2605.11086", "authors": [ "Zhun Wang", "Nico Schiller", "Hongwei Li", "Srijiith Sesha Narayana", "Milad Nasr", "Nicholas Carlini", "Xiangyu Qi", "Eric Wallace", "Elie Bursztein", "Luca Invernizzi", "Kurt Thomas", "Yan Shoshitaishvili", "Wenbo Guo", "Jingxuan He", "Thorsten Holz", "Dawn Song" ], "aliases": [ "https://doi.org/10.48550/arXiv.2605.11086", "https://arxiv.org/abs/2605.11086", "https://arxiv.org/pdf/2605.11086" ] }, { "id": "cve-bench-figure-source", "title": "CVE-Bench: A Benchmark for AI Agents' Ability to Exploit Real-World Web Application Vulnerabilities", "url": "https://arxiv.org/html/2503.17332", "version": "arXiv:2503.17332v4 (2026-10-25 capture)", "publicationDate": "2025-03-21", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2503.17332", "authors": [ "Yuxuan Zhu", "Antony Kellermann", "Dylan Bowman", "Philip Li", "Akul Gupta", "Adarsh Danda", "Richard Fang", "Conner Jensen", "Eric Ihli", "Jason Benn", "Jet Geronimo", "Avi Dhir", "Sudhit Rao", "Kaicheng Yu", "Twm Stone", "Daniel Kang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2503.17332", "https://arxiv.org/abs/2503.17332", "https://arxiv.org/pdf/2503.17332" ] }, { "id": "sre-bench-figure-source", "title": "The Next Challenge for Agentic Cybersecurity: A Realistic, Contamination-Free Reverse Engineering Benchmark", "url": "https://arxiv.org/html/2608.11469", "version": "arXiv:2608.11469v2 (2026-10-25 capture)", "publicationDate": "2026-08-11", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2608.11469", "authors": [ "Jeremy Spence", "Nicholas Assaderaghi", "Feng Xiao", "Jinhao Zhu", "Nikil Ravi", "Xiangyu Qi", "Matthew Jagielski", "Raluca Ada Popa", "Eric Wallace", "Guannan Wei", "Yangruibo Ding", "Zhuo Zhang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2608.11469", "https://arxiv.org/abs/2608.11469", "https://arxiv.org/pdf/2608.11469" ] }, { "id": "hcast-figure-source", "title": "HCAST: Human-Calibrated Autonomy Software Tasks", "url": "https://arxiv.org/html/2503.17354", "version": "arXiv:2503.17354v1 (2026-10-25 capture)", "publicationDate": "2025-03-21", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2503.17354", "authors": [ "David Rein", "Joel Becker", "Amy Deng", "Seraphina Nix", "Chris Canal", "Daniel O’Connell", "Pip Arnott", "Ryan Bloom", "Thomas Broadley", "Katharyn Garcia", "Brian Goodrich", "Max Hasin", "Sami Jawhar", "Megan Kinniment", "Thomas Kwa", "Aron Lajko", "Nate Rush", "Lucas Jun Koba Sato", "Sydney Von Arx", "Ben West", "Lawrence Chan", "Elizabeth Barnes" ], "aliases": [ "https://doi.org/10.48550/arXiv.2503.17354", "https://arxiv.org/abs/2503.17354", "https://arxiv.org/pdf/2503.17354" ] }, { "id": "machiavelli-figure-source", "title": "Do the Rewards Justify the Means? Measuring Trade-Offs Between Rewards and Ethical Behavior in the MACHIAVELLI Benchmark", "url": "https://arxiv.org/html/2304.03279", "version": "arXiv:2304.03279v4 (2026-10-25 capture)", "publicationDate": "2023-04-06", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2304.03279", "authors": [ "Alexander Pan", "Jun Shern Chan", "Andy Zou", "Nathaniel Li", "Steven Basart", "Thomas Woodside", "Jonathan Ng", "Hanlin Zhang", "Scott Emmons", "Dan Hendrycks" ], "aliases": [ "https://doi.org/10.48550/arXiv.2304.03279", "https://arxiv.org/abs/2304.03279", "https://arxiv.org/pdf/2304.03279" ] }, { "id": "tamperbench-figure-source", "title": "TamperBench: Systematically Stress-Testing LLM Safety Under Fine-Tuning and Tampering", "url": "https://arxiv.org/html/2602.06911", "version": "arXiv:2602.06911v2 (2026-10-25 capture)", "publicationDate": "2026-02-06", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2602.06911", "authors": [ "Saad Hossain", "Tom Tseng", "Punya Syon Pandey", "Samanvay Vajpayee", "Matthew Kowal", "Nayeema Nonta", "Samuel Simko", "Stephen Casper", "Zhijing Jin", "Kellin Pelrine", "Sirisha Rambhatla" ], "aliases": [ "https://doi.org/10.48550/arXiv.2602.06911", "https://arxiv.org/abs/2602.06911", "https://arxiv.org/pdf/2602.06911" ] }, { "id": "injecagent-figure-source", "title": "InjecAgent: Benchmarking Indirect Prompt Injections in Tool-Integrated Large Language Model Agents", "url": "https://arxiv.org/html/2403.02691", "version": "arXiv:2403.02691v3 (2026-10-25 capture)", "publicationDate": "2024-03-05", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2403.02691", "authors": [ "Qiusi Zhan", "Zhixiang Liang", "Zifan Ying", "Daniel Kang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2403.02691", "https://arxiv.org/abs/2403.02691", "https://arxiv.org/pdf/2403.02691" ] }, { "id": "agent-safetybench-figure-source", "title": "Agent-SafetyBench: Evaluating the Safety of LLM Agents", "url": "https://arxiv.org/html/2412.14470", "version": "arXiv:2412.14470v2 (2026-10-25 capture)", "publicationDate": "2024-12-19", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2412.14470", "authors": [ "Zhexin Zhang 1 1 1 Equal contribution.", "Shiyao Cui 1 1 1 Equal contribution.", "Yida Lu 1 1 1 Equal contribution.", "Jingzhuo Zhou 1 1 1 Equal contribution.", "Junxiao Yang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2412.14470", "https://arxiv.org/abs/2412.14470", "https://arxiv.org/pdf/2412.14470" ] }, { "id": "labbench-two-figure-source", "title": "LABBench2: An Improved Benchmark for AI Systems Performing Biology Research", "url": "https://arxiv.org/html/2604.09554", "version": "arXiv:2604.09554v2 (2026-10-25 capture)", "publicationDate": "2026-02-04", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2604.09554", "authors": [ "Jon M Laurent", "Albert Bou", "Michael Pieler", "Conor Igoe", "Alex Andonian", "Siddharth Narayanan", "James Braza", "Alexandros Sanchez Vassopoulos", "Jacob L Steenwyk", "Blake Lash", "Andrew D White", "Samuel G Rodriques" ], "aliases": [ "https://doi.org/10.48550/arXiv.2604.09554", "https://arxiv.org/abs/2604.09554", "https://arxiv.org/pdf/2604.09554" ] }, { "id": "fortress-figure-source", "title": "FORTRESS: Frontier Risk Evaluation for National Security and Public Safety", "url": "https://arxiv.org/html/2506.14922", "version": "arXiv:2506.14922v2 (2026-10-25 capture)", "publicationDate": "2025-06-17", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2506.14922", "authors": [ "Christina Q. Knight ∗", "Kaustubh Deshpande ⋄", "Ved Sirdeshmukh ⋄", "Meher Mankikar", "Scale Red Team", "SEAL Research Team", "and Julian Michael" ], "aliases": [ "https://doi.org/10.48550/arXiv.2506.14922", "https://arxiv.org/abs/2506.14922", "https://arxiv.org/pdf/2506.14922" ] }, { "id": "basharena-figure-source", "title": "BashArena: A Control Setting for Highly Privileged AI Agents", "url": "https://arxiv.org/html/2512.15688", "version": "arXiv:2512.15688v1 (2026-10-25 capture)", "publicationDate": "2025-12-17", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2512.15688", "authors": [ "Adam Kaufman *", "James Lucassen *" ], "aliases": [ "https://doi.org/10.48550/arXiv.2512.15688", "https://arxiv.org/abs/2512.15688", "https://arxiv.org/pdf/2512.15688" ] }, { "id": "monitorbench-figure-source", "title": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "url": "https://arxiv.org/html/2603.28590v2", "version": "arXiv:2603.28590v2", "publicationDate": "2026-03-30", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2603.28590", "authors": [ "Han Wang", "Yifan Sun", "Brian Ko", "Mann Talati", "Jiawen Gong", "Zimeng Li   Naicheng Yu   Xucheng Yu   Wei Shen   Vedant Jolly", "Huan Zhang" ], "aliases": [ "https://doi.org/10.48550/arXiv.2603.28590", "https://arxiv.org/abs/2603.28590", "https://arxiv.org/pdf/2603.28590" ] }, { "id": "catastrophic-cyber-benchmark-figure-source", "title": "Catastrophic Cyber Capabilities Benchmark (3CB): Robustly Evaluating LLM Agent Cyber Offense Capabilities", "url": "https://arxiv.org/html/2410.09114", "version": "arXiv:2410.09114v2 (2026-10-25 capture)", "publicationDate": "2024-10-10", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2410.09114", "authors": [ "Your Name" ], "aliases": [ "https://doi.org/10.48550/arXiv.2410.09114", "https://arxiv.org/abs/2410.09114", "https://arxiv.org/pdf/2410.09114" ] }, { "id": "mcptox-figure-source", "title": "MCPTox: A Benchmark for Tool Poisoning Attack on Real-World MCP Servers", "url": "https://arxiv.org/html/2508.14925", "version": "arXiv:2508.14925v2 (2026-10-25 capture)", "publicationDate": "2025-08-19", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2508.14925", "authors": [ "Zhiqiang Wang", "Yichao Gao", "Yanting Wang", "Suyuan Liu", "Haifeng Sun", "Haoran Cheng", "Guanquan Shi", "Haohua Du", "Xiang-Yang Li" ], "aliases": [ "https://doi.org/10.48550/arXiv.2508.14925", "https://arxiv.org/abs/2508.14925", "https://arxiv.org/pdf/2508.14925" ] }, { "id": "narcbench-figure-source", "title": "Detecting Multi-Agent Collusion Through Multi-Agent Interpretability", "url": "https://arxiv.org/html/2604.01151v2", "version": "arXiv:2604.01151v2", "publicationDate": "2026-04-01", "datePrecision": "day", "type": "paper", "verifiedOn": "2026-10-25", "documentId": "arxiv:2604.01151", "authors": [ "Aaron Rose", "Carissa Cullen", "Sahar Abdelnabi", "Philip Torr", "Brandon Gary Kaplowitz", "Christian Schroeder de Witt" ], "aliases": [ "https://doi.org/10.48550/arXiv.2604.01151", "https://arxiv.org/abs/2604.01151", "https://arxiv.org/pdf/2604.01151" ] }, { "id": "strongreject-figure-source", "title": "A StrongREJECT for Empty Jailbreaks", "url": "https://arxiv.org/html/2402.10260v2", "version": "arXiv:2402.10260v2", "publicationDate": "2024-08-27", "datePrecision": "day", "type": "benchmark", "verifiedOn": "2026-10-24", "documentId": "arxiv:2402.10260", "authors": [ "Alexandra Souly ∗", "Qingyuan Lu ∗", "Dillon Bowen ∗", "Tu Trinh †", "Elvis Hsieh †", "Sana Pandey", "Pieter Abbeel", "Justin Svegliato", "Scott Emmons ‡", "Olivia Watkins ‡", "Sam Toyer ‡", "Center for Human-Compatible AI", "UC Berkeley" ], "aliases": [ "https://doi.org/10.48550/arXiv.2402.10260", "https://arxiv.org/abs/2402.10260", "https://arxiv.org/pdf/2402.10260" ] }, { "id": "truthfulqa-figure-source", "title": "TruthfulQA: Measuring how models mimic human falsehoods", "url": "https://arxiv.org/html/2109.07958", "version": "arXiv:2109.07958v2 (2026-10-25 capture)", "publicationDate": "2021-09-08", "datePrecision": "day", "type": "research-publication", "verifiedOn": "2026-10-25", "documentId": "arxiv:2109.07958", "authors": [ "Stephanie Lin", "Jacob Hilton", "Owain Evans" ], "aliases": [ "https://doi.org/10.48550/arXiv.2109.07958", "https://arxiv.org/abs/2109.07958", "https://arxiv.org/pdf/2109.07958" ] }, { "id": "conspiracy-persuasion-figure-source", "title": "Large language models can effectively convince people to believe conspiracies", "url": "https://arxiv.org/html/2601.05050v3", "version": "arXiv:2601.05050v3", "publicationDate": "2026-01-08", "datePrecision": "day", "type": "controlled-study", "verifiedOn": "2026-10-25", "documentId": "arxiv:2601.05050" }, { "id": "palisade-robot-technical-report", "title": "Shutdown Resistance in Large Language Models, on robots!", "url": "https://palisaderesearch.org/assets/reports/shutdown-resistance-on-robots.pdf", "version": "Technical report · 2026-02-11", "publicationDate": "2026-02-11", "datePrecision": "day", "type": "technical-report", "verifiedOn": "2026-10-25", "authors": [ "Artem Petrov", "Sergey Koldyba", "Sergey Molchanov", "Nikolaj Kotov", "Dmitrii Volkov", "Oleg Serikov" ] }, { "id": "neo-kimi-k3-source", "title": "Evaluating Kimi K3 for Misalignment Risks", "url": "https://neoresearch.ai/papers/Kimi_K3_Safety_Evaluation.pdf", "version": "September 2026", "publicationDate": "2026-09-11", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "https://neoresearch.ai/papers/Kimi_K3_Safety_Evaluation.pdf" }, { "id": "neo-deepseek-v4-source", "title": "Evaluating DeepSeek v4 Pro for Frontier Risks", "url": "https://neoresearch.ai/papers/DSv4_Safety_Evaluation_v1.1.pdf", "version": "v1.1 · 29 May 2026", "publicationDate": "2026-05-29", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "https://neoresearch.ai/papers/DSv4_Safety_Evaluation_v1.1.pdf" }, { "id": "neo-glm52-source", "title": "Early Results on Evaluating GLM 5.2 for Behavioral and Persuasion Risks", "url": "https://neoresearch.ai/research/glm-5-2-behavioral-persuasion-evaluation/", "version": "3 July 2026", "publicationDate": "2026-07-03", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "https://neoresearch.ai/research/glm-5-2-behavioral-persuasion-evaluation" }, { "id": "saferai-glm52-source", "title": "GLM-5.2 Risk Evaluation Report", "url": "https://www.safer-ai.org/u/2026/08/SaferAI-GLM-Evaluation-Report.pdf", "version": "v1.1 · revised 26 August 2026", "publicationDate": "2026-08-02", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "https://www.safer-ai.org/u/2026/08/SaferAI-GLM-Evaluation-Report.pdf" }, { "id": "caisi-deepseek-2025-source", "title": "Evaluation of DeepSeek AI Models", "url": "https://www.nist.gov/system/files/documents/2025/09/30/CAISI_Evaluation_of_DeepSeek_AI_Models.pdf", "version": "30 September 2025", "publicationDate": "2025-09-30", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "https://www.nist.gov/system/files/documents/2025/09/30/CAISI_Evaluation_of_DeepSeek_AI_Models.pdf" }, { "id": "caisi-deepseek-v4-source", "title": "CAISI Evaluation of DeepSeek V4 Pro", "url": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "version": "1 May 2026", "publicationDate": "2026-05-01", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro" }, { "id": "abstentionbench-primary", "title": "AbstentionBench", "url": "https://arxiv.org/abs/2506.09038", "version": "2026-10-25 source capture", "publicationDate": "2025-06-10", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "arxiv:2506.09038" }, { "id": "deceptionbench-primary", "title": "DeceptionBench", "url": "https://arxiv.org/abs/2510.15501", "version": "2026-10-25 source capture", "publicationDate": "2025-10-17", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "arxiv:2510.15501" }, { "id": "persusafety-primary", "title": "PersuSafety", "url": "https://arxiv.org/abs/2504.10430", "version": "2026-10-25 source capture", "publicationDate": "2025-04-14", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "arxiv:2504.10430" }, { "id": "impossiblebench-primary", "title": "ImpossibleBench", "url": "https://arxiv.org/abs/2510.20270", "version": "2026-10-25 source capture", "publicationDate": "2025-10-23", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "arxiv:2510.20270" }, { "id": "evalawarebench-primary", "title": "EvalAwareBench", "url": "https://arxiv.org/abs/2605.23055", "version": "2026-10-25 source capture", "publicationDate": null, "datePrecision": "unknown", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "arxiv:2605.23055" }, { "id": "theagentcompany-primary", "title": "TheAgentCompany", "url": "https://arxiv.org/abs/2412.14161", "version": "2026-10-25 source capture", "publicationDate": "2024-12-18", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "arxiv:2412.14161" }, { "id": "lab-bench-primary", "title": "LAB-Bench", "url": "https://arxiv.org/abs/2407.10362", "version": "2026-10-25 source capture", "publicationDate": "2024-07-14", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "arxiv:2407.10362" }, { "id": "charxiv-primary", "title": "CharXiv", "url": "https://charxiv.github.io/", "version": "2026-10-25 source capture", "publicationDate": null, "datePrecision": "unknown", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "https://charxiv.github.io" }, { "id": "petri-primary", "title": "Petri", "url": "https://www.anthropic.com/research/petri-open-source-auditing", "version": "2026-10-25 source capture", "publicationDate": null, "datePrecision": "unknown", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "https://www.anthropic.com/research/petri-open-source-auditing" }, { "id": "biomysterybench-primary", "title": "BioMysteryBench", "url": "https://www.anthropic.com/research/Evaluating-Claude-For-Bioinformatics-With-BioMysteryBench", "version": "2026-10-25 source capture", "publicationDate": null, "datePrecision": "unknown", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "https://www.anthropic.com/research/Evaluating-Claude-For-Bioinformatics-With-BioMysteryBench" }, { "id": "evmbench-primary", "title": "EVMbench", "url": "https://openai.com/index/introducing-evmbench/", "version": "2026-10-25 source capture", "publicationDate": null, "datePrecision": "unknown", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "https://openai.com/index/introducing-evmbench" }, { "id": "swe-bench-pro-primary", "title": "SWE-Bench Pro", "url": "https://arxiv.org/abs/2509.16941", "version": "2026-10-25 source capture", "publicationDate": null, "datePrecision": "unknown", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "arxiv:2509.16941" }, { "id": "ctf-archive-diamond-primary", "title": "CTF-Archive-Diamond", "url": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "version": "2026-10-25 source capture", "publicationDate": "2026-05-01", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro" }, { "id": "portbench-primary", "title": "PortBench", "url": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "version": "2026-10-25 source capture", "publicationDate": "2026-05-01", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro" }, { "id": "metr-horizon-11-data", "title": "METR Time Horizon 1.1 — source data", "url": "https://metr.org/assets/benchmark_results_1_1.yaml", "version": "METR-Horizon-v1.1 · 2026-10-25 capture", "publicationDate": null, "datePrecision": "unknown", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "https://metr.org/assets/benchmark_results_1_1.yaml" }, { "id": "metr-horizon-11-page", "title": "Task-Completion Time Horizons of Frontier AI Models", "url": "https://metr.org/time-horizons/", "version": "Page last updated 8 September 2026; captured 25 October 2026", "publicationDate": "2026-09-08", "datePrecision": "day", "type": "original-report", "verifiedOn": "2026-10-25", "authors": [], "documentId": "https://metr.org/time-horizons" }, { "id": "official-anthropic-anthropic-february-2026-risk-report", "title": "Anthropic February 2026 Risk Report", "url": "https://anthropic.com/feb-2026-risk-report", "version": "2026-07-08", "publicationDate": null, "datePrecision": "unknown", "type": "risk-report", "verifiedOn": "2026-10-25", "documentId": "https://anthropic.com/feb-2026-risk-report" }, { "id": "official-anthropic-claude-2-system-card", "title": "Claude 2 System Card", "url": "https://www-cdn.anthropic.com/bd2a28d2535bfb0494cc8e2a3bf135d2e7523226/Model-Card-Claude-2.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www-cdn.anthropic.com/bd2a28d2535bfb0494cc8e2a3bf135d2e7523226/Model-Card-Claude-2.pdf" }, { "id": "official-anthropic-claude-3-system-card", "title": "Claude 3 System Card", "url": "https://www-cdn.anthropic.com/c6a80a657af445f40e31afac050f3bf76d3b1404.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www-cdn.anthropic.com/c6a80a657af445f40e31afac050f3bf76d3b1404.pdf" }, { "id": "official-anthropic-claude-haiku-3-5-and-sonnet-3-5-new-system-card", "title": "Claude Haiku 3.5 and Sonnet 3.5 (new) System Card", "url": "https://www-cdn.anthropic.com/c7822cdc35ad788ec87e14b3a9d45010f1f86c38.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www-cdn.anthropic.com/c7822cdc35ad788ec87e14b3a9d45010f1f86c38.pdf" }, { "id": "official-anthropic-claude-sonnet-3-5-system-card", "title": "Claude Sonnet 3.5 System Card", "url": "https://www-cdn.anthropic.com/fed9cc193a14b84131812372d8d5857f8f304c52/Model_Card_Claude_3_Addendum.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www-cdn.anthropic.com/fed9cc193a14b84131812372d8d5857f8f304c52/Model_Card_Claude_3_Addendum.pdf" }, { "id": "official-anthropic-claude-sonnet-3-7-system-card", "title": "Claude Sonnet 3.7 System Card", "url": "https://www-cdn.anthropic.com/9ff93dfa8f445c932415d335c88852ef47f1201e.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www-cdn.anthropic.com/9ff93dfa8f445c932415d335c88852ef47f1201e.pdf" }, { "id": "official-anthropic-claude-sonnet-4-and-opus-4-system-card", "title": "Claude Sonnet 4 and Opus 4 System Card", "url": "https://www-cdn.anthropic.com/07b2a3f9902ee19fe39a36ca638e5ae987bc64dd.pdf", "version": "2025-05-04", "publicationDate": "2025-05-04", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www-cdn.anthropic.com/07b2a3f9902ee19fe39a36ca638e5ae987bc64dd.pdf" }, { "id": "official-anthropic-claude-opus-4-1-system-card", "title": "Claude Opus 4.1 System Card", "url": "https://www-cdn.anthropic.com/9fa30625273bafdf5af82c93719d7ca606485a16.pdf", "version": "2025-09-15", "publicationDate": "2025-08-01", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www-cdn.anthropic.com/9fa30625273bafdf5af82c93719d7ca606485a16.pdf" }, { "id": "official-anthropic-claude-sonnet-4-5-system-card", "title": "Claude Sonnet 4.5 System Card", "url": "https://www-cdn.anthropic.com/963373e433e489a87a10c823c52a0a013e9172dd.pdf", "version": "2025-12-03", "publicationDate": "2025-09-05", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www-cdn.anthropic.com/963373e433e489a87a10c823c52a0a013e9172dd.pdf" }, { "id": "official-anthropic-claude-haiku-4-5-system-card", "title": "Claude Haiku 4.5 System Card", "url": "https://www-cdn.anthropic.com/7aad69bf12627d42234e01ee7c36305dc2f6a970.pdf", "version": "2025-10-05", "publicationDate": "2025-10-05", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www-cdn.anthropic.com/7aad69bf12627d42234e01ee7c36305dc2f6a970.pdf" }, { "id": "official-anthropic-claude-opus-4-5-system-card", "title": "Claude Opus 4.5 System Card", "url": "https://www-cdn.anthropic.com/bf10f64990cfda0ba858290be7b8cc6317685f47.pdf", "version": "2025-12-05", "publicationDate": "2025-11-05", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www-cdn.anthropic.com/bf10f64990cfda0ba858290be7b8cc6317685f47.pdf" }, { "id": "official-anthropic-claude-opus-4-6-system-card-retry", "title": "Claude Opus 4.6 System Card", "url": "https://anthropic.com/claude-opus-4-6-system-card", "version": "2026-03-06", "publicationDate": "2026-02-06", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://anthropic.com/claude-opus-4-6-system-card" }, { "id": "official-anthropic-claude-opus-4-6-sabotage-risk-report", "title": "Claude Opus 4.6 Sabotage Risk Report", "url": "https://anthropic.com/claude-opus-4-6-risk-report", "version": "2026-02-10", "publicationDate": "2026-02-10", "datePrecision": "day", "type": "risk-report", "verifiedOn": "2026-10-25", "documentId": "https://anthropic.com/claude-opus-4-6-risk-report" }, { "id": "official-anthropic-claude-sonnet-4-6-system-card", "title": "Claude Sonnet 4.6 System Card", "url": "https://anthropic.com/claude-sonnet-4-6-system-card", "version": "2026-03-06", "publicationDate": "2026-02-17", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://anthropic.com/claude-sonnet-4-6-system-card" }, { "id": "official-anthropic-claude-mythos-preview-risk-report", "title": "Alignment Risk Update: Claude Mythos Preview", "url": "https://www.anthropic.com/claude-mythos-preview-risk-report", "version": "2026-04-10", "publicationDate": "2026-04-07", "datePrecision": "day", "type": "risk-report", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/claude-mythos-preview-risk-report" }, { "id": "official-anthropic-mythos-preview-system-card", "title": "Mythos Preview System Card", "url": "https://www.anthropic.com/claude-mythos-preview-system-card", "version": "2026-04-14", "publicationDate": "2026-04-07", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/claude-mythos-preview-system-card" }, { "id": "official-anthropic-claude-opus-4-7-system-card", "title": "Claude Opus 4.7 System Card", "url": "https://anthropic.com/claude-opus-4-7-system-card", "version": "2026-04-16", "publicationDate": "2026-04-16", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://anthropic.com/claude-opus-4-7-system-card" }, { "id": "official-anthropic-claude-opus-4-8-system-card", "title": "Claude Opus 4.8 System Card", "url": "https://anthropic.com/claude-opus-4-8-system-card", "version": "2026-06-17", "publicationDate": "2026-05-28", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://anthropic.com/claude-opus-4-8-system-card" }, { "id": "official-anthropic-claude-fable-5-and-mythos-5-system-card", "title": "Claude Fable 5 and Mythos 5 System Card", "url": "https://anthropic.com/claude-fable-5-mythos-5-system-card", "version": "2026-06-25", "publicationDate": "2026-06-09", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://anthropic.com/claude-fable-5-mythos-5-system-card" }, { "id": "official-anthropic-claude-sonnet-5-system-card", "title": "Claude Sonnet 5 System Card", "url": "https://anthropic.com/claude-sonnet-5-system-card", "version": "2026-07-10", "publicationDate": "2026-06-30", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://anthropic.com/claude-sonnet-5-system-card" }, { "id": "official-anthropic-claude-opus-5-system-card", "title": "Claude Opus 5 System Card", "url": "https://anthropic.com/claude-opus-5-system-card", "version": "2026-08-19", "publicationDate": "2026-07-24", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://anthropic.com/claude-opus-5-system-card" }, { "id": "official-anthropic-anthropic-august-2026-risk-report", "title": "Anthropic August 2026 Risk Report", "url": "https://anthropic.com/aug-2026-risk-report", "version": "2026-08-14", "publicationDate": "2026-08-14", "datePrecision": "day", "type": "risk-report", "verifiedOn": "2026-10-25", "documentId": "https://anthropic.com/aug-2026-risk-report" }, { "id": "official-anthropic-claude-fable-5-1-and-mythos-5-1-system-card", "title": "Claude Fable 5.1 and Mythos 5.1 System Card", "url": "https://www.anthropic.com/claude-fable-5-1-mythos-5-1-system-card", "version": "2026-09-01", "publicationDate": "2026-09-01", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/claude-fable-5-1-mythos-5-1-system-card" }, { "id": "official-anthropic-claude-opus-5-5-system-card", "title": "Claude Opus 5.5 System Card", "url": "https://www.anthropic.com/claude-opus-5-5-system-card", "version": "2026-09-22", "publicationDate": "2026-09-22", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/claude-opus-5-5-system-card" }, { "id": "official-anthropic-claude-sonnet-5-5-system-card", "title": "Claude Sonnet 5.5 System Card", "url": "https://www.anthropic.com/claude-sonnet-5-5-system-card", "version": "2026-09-28", "publicationDate": "2026-09-28", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/claude-sonnet-5-5-system-card" }, { "id": "official-anthropic-claude-haiku-5-5-system-card", "title": "Claude Haiku 5.5 System Card", "url": "https://www.anthropic.com/claude-haiku-5-5-system-card", "version": "2026-10-07", "publicationDate": "2026-10-07", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://www.anthropic.com/claude-haiku-5-5-system-card" }, { "id": "official-deepseek-deepseek-llm-scaling-open-source-language-models-with-longtermism", "title": "DeepSeek LLM: Scaling Open-Source Language Models with Longtermism", "url": "https://arxiv.org/html/2311.18743", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2311.18743" }, { "id": "official-deepseek-deepseek-r1-model-card", "title": "DeepSeek-R1 Model Card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/deepseek-ai/DeepSeek-R1" }, { "id": "official-deepseek-deepseek-r1-0528-model-card", "title": "DeepSeek-R1-0528 Model Card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/deepseek-ai/DeepSeek-R1-0528" }, { "id": "official-deepseek-deepseek-r1-incentivizing-reasoning-capability-in-llms-via-reinforcement-learning", "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning", "url": "https://arxiv.org/html/2501.12948", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2501.12948" }, { "id": "official-deepseek-deepseek-v2-model-card", "title": "DeepSeek-V2 Model Card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V2", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/deepseek-ai/DeepSeek-V2" }, { "id": "official-deepseek-deepseek-v2-a-strong-economical-and-efficient-mixture-of-experts-language-model", "title": "DeepSeek-V2: A Strong, Economical, and Efficient Mixture-of-Experts Language Model", "url": "https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V2/main/deepseek-v2-tech-report.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "https://raw.githubusercontent.com/deepseek-ai/DeepSeek-V2/main/deepseek-v2-tech-report.pdf" }, { "id": "official-deepseek-deepseek-v3-model-card", "title": "DeepSeek-V3 Model Card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/deepseek-ai/DeepSeek-V3" }, { "id": "official-deepseek-deepseek-v3-technical-report", "title": "DeepSeek-V3 Technical Report", "url": "https://arxiv.org/html/2412.19437", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2412.19437" }, { "id": "official-deepseek-deepseek-v3-1-model-card", "title": "DeepSeek-V3.1 Model Card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/deepseek-ai/DeepSeek-V3.1" }, { "id": "official-deepseek-deepseek-v3-2-model-card", "title": "DeepSeek-V3.2 Model Card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2" }, { "id": "official-deepseek-deepseek-v3-2-exp-model-card", "title": "DeepSeek-V3.2-Exp Model Card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2-Exp" }, { "id": "official-deepseek-deepseek-v3-2-efficient-reasoning-agentic-ai", "title": "DeepSeek-V3.2: Efficient Reasoning & Agentic AI", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/main/assets/paper.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/deepseek-ai/DeepSeek-V3.2/resolve/main/assets/paper.pdf" }, { "id": "official-deepseek-deepseek-v4-flash-model-card", "title": "DeepSeek-V4-Flash Model Card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash" }, { "id": "official-deepseek-deepseek-v4-pro-model-card", "title": "DeepSeek-V4-Pro Model Card", "url": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/deepseek-ai/DeepSeek-V4-Pro" }, { "id": "official-deepseek-deepseek-v4-towards-highly-efficient-million-token-context-intelligence", "title": "DeepSeek-V4: Towards Highly Efficient Million-Token Context Intelligence", "url": "https://arxiv.org/html/2606.19348", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2606.19348" }, { "id": "official-deepseek-deepseek-llm-67b-chat-model-card", "title": "deepseek-llm-67b-chat Model Card", "url": "https://huggingface.co/deepseek-ai/deepseek-llm-67b-chat", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/deepseek-ai/deepseek-llm-67b-chat" }, { "id": "official-google-deepmind-codegemma-model-card", "title": "CodeGemma Model Card", "url": "https://ai.google.dev/gemma/docs/codegemma/model_card", "version": "2025-02-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/codegemma/model_card" }, { "id": "official-google-deepmind-diffusiongemma-model-card", "title": "DiffusionGemma Model Card", "url": "https://ai.google.dev/gemma/docs/diffusiongemma/model_card?utm_source=deepmind.google&utm_medium=referral&utm_campaign=gdm&utm_content", "version": "2026-06-10", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/diffusiongemma/model_card" }, { "id": "official-google-deepmind-embeddinggemma-2-model-card", "title": "EmbeddingGemma 2 Model Card", "url": "https://ai.google.dev/gemma/docs/embeddinggemma/model_card_2?utm_source=deepmind.google&utm_medium=referral&utm_campaign=gdm&utm_content", "version": "2026-10-06", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/embeddinggemma/model_card_2" }, { "id": "official-google-deepmind-embeddinggemma-model-card", "title": "EmbeddingGemma Model Card", "url": "https://ai.google.dev/gemma/docs/embeddinggemma/model_card", "version": "2025-09-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/embeddinggemma/model_card" }, { "id": "official-google-deepmind-functiongemma-model-card", "title": "FunctionGemma Model Card", "url": "https://ai.google.dev/gemma/docs/functiongemma/model_card", "version": "2026-01-14", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/functiongemma/model_card" }, { "id": "official-google-deepmind-gemini-1-0-model-card", "title": "Gemini 1.0 Model Card", "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_1_report.pdf#page=71", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/gemini/gemini_1_report.pdf" }, { "id": "official-google-deepmind-gemini-1-5-model-card", "title": "Gemini 1.5 Model Card", "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf#page=105", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v1_5_report.pdf" }, { "id": "official-google-deepmind-gemini-2-0-flash-model-card", "title": "Gemini 2.0 Flash Model Card", "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-0-Flash-Model-Card.pdf", "version": "2025-04-15", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-0-Flash-Model-Card.pdf" }, { "id": "official-google-deepmind-gemini-2-0-flash-lite-model-card", "title": "Gemini 2.0 Flash-Lite Model Card", "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-0-Flash-Lite-Model-Card.pdf", "version": "2025-04-10", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-0-Flash-Lite-Model-Card.pdf" }, { "id": "official-google-deepmind-gemini-2-5-computer-use-model-card", "title": "Gemini 2.5 Computer Use Model Card", "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Computer-Use-Model-Card.pdf", "version": "2025-10-07", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Computer-Use-Model-Card.pdf" }, { "id": "official-google-deepmind-gemini-2-5-deep-think-model-card", "title": "Gemini 2.5 Deep Think Model Card", "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Deep-Think-Model-Card.pdf", "version": "2025-08-01", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Deep-Think-Model-Card.pdf" }, { "id": "official-google-deepmind-gemini-2-5-flash-and-gemini-2-5-flash-image-model-card", "title": "Gemini 2.5 Flash and Gemini 2.5 Flash Image Model Card", "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Flash-Model-Card.pdf", "version": "2025-09-26", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Flash-Model-Card.pdf" }, { "id": "official-google-deepmind-gemini-2-5-flash-lite-model-card", "title": "Gemini 2.5 Flash-Lite Model Card", "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Flash-Lite-Model-Card.pdf", "version": "2025-09-26", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Flash-Lite-Model-Card.pdf" }, { "id": "official-google-deepmind-gemini-2-5-pro-model-card", "title": "Gemini 2.5 Pro Model Card", "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Pro-Model-Card.pdf", "version": "2025-06-27", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-2-5-Pro-Model-Card.pdf" }, { "id": "official-google-deepmind-gemini-2-5-pushing-the-frontier-with-advanced-reasoning-multimodality-long-context-and-next-generation-ag", "title": "Gemini 2.5: Pushing the Frontier with Advanced Reasoning, Multimodality, Long Context, and Next Generation Agentic Capabilities", "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/gemini/gemini_v2_5_report.pdf" }, { "id": "official-google-deepmind-gemini-3-flash-model-card", "title": "Gemini 3 Flash Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-flash/", "version": "2025-12-17", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-flash" }, { "id": "official-google-deepmind-gemini-3-pro-image-model-card", "title": "Gemini 3 Pro Image Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-pro-image/", "version": "2025-11-20", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-pro-image" }, { "id": "official-google-deepmind-gemini-3-pro-model-card", "title": "Gemini 3 Pro Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-pro/", "version": "2025-11-18", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-pro" }, { "id": "official-google-deepmind-gemini-4-argon-safety-security-report", "title": "Gemini 4 Argon Safety & Security Report", "url": "https://storage.googleapis.com/deepmind-media/gemini/gemini_4_safety_report.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "safety-security-report", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/gemini/gemini_4_safety_report.pdf" }, { "id": "official-google-deepmind-gemini-robotics-1-5-model-card", "title": "Gemini Robotics 1.5 Model Card", "url": "https://storage.googleapis.com/deepmind-media/gemini-robotics/Gemini-Robotics-1-5-Tech-Report.pdf#page=30", "version": "2025-09-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/gemini-robotics/Gemini-Robotics-1-5-Tech-Report.pdf" }, { "id": "official-google-deepmind-gemini-robotics-on-device-model-card", "title": "Gemini Robotics On-Device Model Card", "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-Robotics-On-Device-Model-Card.pdf", "version": "2025-07-01", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/Model-Cards/Gemini-Robotics-On-Device-Model-Card.pdf" }, { "id": "official-google-deepmind-gemma-1-model-card", "title": "Gemma 1 Model Card", "url": "https://ai.google.dev/gemma/docs/model_card", "version": "2025-02-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/model_card" }, { "id": "official-google-deepmind-gemma-2-model-card", "title": "Gemma 2 Model Card", "url": "https://ai.google.dev/gemma/docs/model_card_2", "version": "2025-02-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/model_card_2" }, { "id": "official-google-deepmind-gemma-3-model-card", "title": "Gemma 3 Model Card", "url": "https://ai.google.dev/gemma/docs/core/model_card_3", "version": "2025-08-14", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/core/model_card_3" }, { "id": "official-google-deepmind-gemma-3n-model-card", "title": "Gemma 3n Model Card", "url": "https://ai.google.dev/gemma/docs/gemma-3n/model_card", "version": "2025-06-17", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/gemma-3n/model_card" }, { "id": "official-google-deepmind-gemma-4-model-card", "title": "Gemma 4 Model Card", "url": "https://ai.google.dev/gemma/docs/core/model_card_4?utm_source=deepmind.google&utm_medium=referral&utm_campaign=gdm&utm_content", "version": "2026-04-02", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/core/model_card_4" }, { "id": "official-google-deepmind-imagen-4-model-card", "title": "Imagen 4 Model Card", "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Imagen-4-Model-Card.pdf", "version": "2025-05-20", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/Model-Cards/Imagen-4-Model-Card.pdf" }, { "id": "official-google-deepmind-paligemma-1-model-card", "title": "PaliGemma 1 Model Card", "url": "https://ai.google.dev/gemma/docs/paligemma/model-card", "version": "2025-02-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/paligemma/model-card" }, { "id": "official-google-deepmind-paligemma-2-model-card", "title": "PaliGemma 2 Model Card", "url": "https://ai.google.dev/gemma/docs/paligemma/model-card-2", "version": "2025-02-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/paligemma/model-card-2" }, { "id": "official-google-deepmind-recurrentgemma-model-card", "title": "RecurrentGemma Model Card", "url": "https://ai.google.dev/gemma/docs/recurrentgemma/model_card", "version": "2025-02-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/recurrentgemma/model_card" }, { "id": "official-google-deepmind-shieldgemma-1-model-card", "title": "ShieldGemma 1 Model Card", "url": "https://ai.google.dev/gemma/docs/shieldgemma/model_card", "version": "2025-02-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/shieldgemma/model_card" }, { "id": "official-google-deepmind-shieldgemma-2-model-card", "title": "ShieldGemma 2 Model Card", "url": "https://ai.google.dev/gemma/docs/shieldgemma/model_card_2", "version": "2025-06-17", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://ai.google.dev/gemma/docs/shieldgemma/model_card_2" }, { "id": "official-google-deepmind-veo-3-model-card", "title": "Veo 3 Model Card", "url": "https://storage.googleapis.com/deepmind-media/Model-Cards/Veo-3-Model-Card.pdf", "version": "2026-01-13", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://storage.googleapis.com/deepmind-media/Model-Cards/Veo-3-Model-Card.pdf" }, { "id": "official-google-deepmind-lyria-3-model-card", "title": "Lyria 3 Model Card", "url": "https://deepmind.google/models/model-cards/lyria-3/", "version": "2026-02-18", "publicationDate": "2026-02-18", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/lyria-3" }, { "id": "official-google-deepmind-gemini-3-1-pro-model-card", "title": "Gemini 3.1 Pro Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-1-pro/", "version": "2026-02-19", "publicationDate": "2026-02-19", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-1-pro" }, { "id": "official-google-deepmind-gemini-3-1-flash-image-model-card", "title": "Gemini 3.1 Flash Image Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-1-flash-image/", "version": "2026-02-26", "publicationDate": "2026-02-26", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-1-flash-image" }, { "id": "official-google-deepmind-gemini-3-1-flash-lite-model-card", "title": "Gemini 3.1 Flash-Lite Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-1-flash-lite/", "version": "2026-03-03", "publicationDate": "2026-03-03", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-1-flash-lite" }, { "id": "official-google-deepmind-gemini-3-1-flash-audio-flash-live-tts-model-card", "title": "Gemini 3.1 Flash Audio (Flash Live, TTS) Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-1-flash-audio/", "version": "2026-04-15", "publicationDate": "2026-03-26", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-1-flash-audio" }, { "id": "official-google-deepmind-veo-3-1-lite-model-card", "title": "Veo 3.1 Lite Model Card", "url": "https://deepmind.google/models/model-cards/veo-3-1-lite/", "version": "2026-04-08", "publicationDate": "2026-04-08", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/veo-3-1-lite" }, { "id": "official-google-deepmind-gemini-robotics-er-1-6-model-card", "title": "Gemini Robotics-ER 1.6 Model Card", "url": "https://deepmind.google/models/model-cards/gemini-robotics-er-1-6/", "version": "2026-04-20", "publicationDate": "2026-04-20", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-robotics-er-1-6" }, { "id": "official-google-deepmind-gemini-3-5-flash-model-card", "title": "Gemini 3.5 Flash Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-5-flash/", "version": "2026-05-19", "publicationDate": "2026-05-19", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-5-flash" }, { "id": "official-google-deepmind-gemini-3-1-flash-lite-image-model-card", "title": "Gemini 3.1 Flash-Lite Image Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-1-flash-lite-image/", "version": "2026-06-30", "publicationDate": "2026-06-30", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-1-flash-lite-image" }, { "id": "official-google-deepmind-gemini-3-5-flash-lite-model-card", "title": "Gemini 3.5 Flash-Lite Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-5-flash-lite/", "version": "2026-07-21", "publicationDate": "2026-07-21", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-5-flash-lite" }, { "id": "official-google-deepmind-gemini-3-6-flash-model-card", "title": "Gemini 3.6 Flash Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-6-flash/", "version": "2026-07-21", "publicationDate": "2026-07-21", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-6-flash" }, { "id": "official-google-deepmind-lyria-3-5-model-card", "title": "Lyria 3.5 Model Card", "url": "https://deepmind.google/models/model-cards/lyria-3-5/", "version": "2026-07-29", "publicationDate": "2026-07-29", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/lyria-3-5" }, { "id": "official-google-deepmind-gemini-robotics-on-device-2-model-card", "title": "Gemini Robotics On-Device 2 Model Card", "url": "https://deepmind.google/models/model-cards/gemini-robotics-on-device-2/", "version": "2026-07-30", "publicationDate": "2026-07-30", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-robotics-on-device-2" }, { "id": "official-google-deepmind-gemini-robotics-er-2-model-card", "title": "Gemini Robotics-ER 2 Model Card", "url": "https://deepmind.google/models/model-cards/gemini-robotics-er-2/", "version": "2026-07-30", "publicationDate": "2026-07-30", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-robotics-er-2" }, { "id": "official-google-deepmind-gemini-3-7-flash-model-card", "title": "Gemini 3.7 Flash Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-7-flash/", "version": "2026-08-13", "publicationDate": "2026-08-13", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-7-flash" }, { "id": "official-google-deepmind-gemini-3-5-audio-live-translate-transcribe-transcribe-live-model-card", "title": "Gemini 3.5 Audio (Live Translate, Transcribe, Transcribe Live) Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-5-audio/", "version": "2026-08-26", "publicationDate": "2026-08-26", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-5-audio" }, { "id": "official-google-deepmind-gemini-omni-flash-model-card", "title": "Gemini Omni Flash Model Card", "url": "https://deepmind.google/models/model-cards/gemini-omni-flash/", "version": "2026-08-27", "publicationDate": "2026-08-27", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-omni-flash" }, { "id": "official-google-deepmind-gemini-3-8-flash-model-card", "title": "Gemini 3.8 Flash Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-8-flash/", "version": "2026-09-02", "publicationDate": "2026-09-02", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-8-flash" }, { "id": "official-google-deepmind-gemini-3-8-audio-live-live-extended-thinking-flash-tts-flash-lite-tts-model-card", "title": "Gemini 3.8 Audio (Live, Live Extended Thinking, Flash TTS, Flash-Lite TTS) Model Card", "url": "https://deepmind.google/models/model-cards/gemini-3-8-audio/", "version": "2026-09-24", "publicationDate": "2026-09-15", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-3-8-audio" }, { "id": "official-google-deepmind-nano-banana-2-1-model-card", "title": "Nano Banana 2.1 Model Card", "url": "https://deepmind.google/models/model-cards/nano-banana-2-1/", "version": "2026-10-06", "publicationDate": "2026-10-06", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/nano-banana-2-1" }, { "id": "official-google-deepmind-gemini-4-argon-model-card", "title": "Gemini 4 Argon Model Card", "url": "https://deepmind.google/models/model-cards/gemini-4-argon/", "version": "2026-10-13", "publicationDate": "2026-10-13", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://deepmind.google/models/model-cards/gemini-4-argon" }, { "id": "official-meta-llama-2-open-foundation-and-fine-tuned-chat-models", "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models", "url": "https://arxiv.org/html/2307.09288", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2307.09288" }, { "id": "official-meta-llama-2-70b-chat-hf-model-card", "title": "Llama-2-70b-chat-hf Model Card", "url": "https://huggingface.co/meta-llama/Llama-2-70b-chat-hf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/meta-llama/Llama-2-70b-chat-hf" }, { "id": "official-meta-llama-3-1-405b-instruct-model-card", "title": "Llama-3.1-405B-Instruct Model Card", "url": "https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct" }, { "id": "official-meta-llama-3-2-90b-vision-instruct-model-card", "title": "Llama-3.2-90B-Vision-Instruct Model Card", "url": "https://huggingface.co/meta-llama/Llama-3.2-90B-Vision-Instruct", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/meta-llama/Llama-3.2-90B-Vision-Instruct" }, { "id": "official-meta-llama-3-3-70b-instruct-model-card", "title": "Llama-3.3-70B-Instruct Model Card", "url": "https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/meta-llama/Llama-3.3-70B-Instruct" }, { "id": "official-meta-llama-4-maverick-17b-128e-instruct-model-card", "title": "Llama-4-Maverick-17B-128E-Instruct Model Card", "url": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/meta-llama/Llama-4-Maverick-17B-128E-Instruct" }, { "id": "official-meta-llama-4-scout-17b-16e-instruct-model-card", "title": "Llama-4-Scout-17B-16E-Instruct Model Card", "url": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/meta-llama/Llama-4-Scout-17B-16E-Instruct" }, { "id": "official-meta-meta-llama-3-70b-instruct-model-card", "title": "Meta-Llama-3-70B-Instruct Model Card", "url": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/meta-llama/Meta-Llama-3-70B-Instruct" }, { "id": "official-meta-muse-glimmer-model-card", "title": "Muse Glimmer Model Card", "url": "https://huggingface.co/meta-models/Muse-Glimmer-30B", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/meta-models/Muse-Glimmer-30B" }, { "id": "official-meta-muse-spark-1-2-evaluation-report", "title": "Muse Spark 1.2 Multimodal Evaluation Methodology", "url": "https://research.meta.ai/static/muse-spark-1-2-multimodal-evaluation-methodology", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "evaluation-methodology", "verifiedOn": "2026-10-25", "documentId": "https://research.meta.ai/static/muse-spark-1-2-multimodal-evaluation-methodology" }, { "id": "official-meta-muse-spark-1-3-evaluation-report", "title": "Muse Spark 1.3 Evaluation Methodology", "url": "https://research.meta.ai/static/muse-spark-1-3-multimodal-evaluation-methodology", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "evaluation-methodology", "verifiedOn": "2026-10-25", "documentId": "https://research.meta.ai/static/muse-spark-1-3-multimodal-evaluation-methodology" }, { "id": "official-meta-muse-spark-contemplating-safety-preparedness-report", "title": "Muse Spark Contemplating Safety & Preparedness Report", "url": "https://ai.meta.com/static-resource/muse-spark-contemplating-safety-and-preparedness-report/", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "safety-preparedness-report", "verifiedOn": "2026-10-25", "documentId": "https://ai.meta.com/static-resource/muse-spark-contemplating-safety-and-preparedness-report" }, { "id": "official-meta-muse-spark-open-evaluation-methodology", "title": "Muse Spark Open Evaluation Methodology", "url": "https://research.meta.ai/static/muse-spark-open-evaluation-methodology", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "evaluation-methodology", "verifiedOn": "2026-10-25", "documentId": "https://research.meta.ai/static/muse-spark-open-evaluation-methodology" }, { "id": "official-meta-muse-spark-open-model-card", "title": "Muse Spark Open Model Card", "url": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/meta-models/Muse-Spark-Open-816B-A42B" }, { "id": "official-meta-muse-spark-safety-preparedness-report-meta-original", "title": "Muse Spark Safety & Preparedness Report (Meta-hosted May 26 revision)", "url": "https://ai.meta.com/static-resource/muse-spark-safety-and-preparedness-report/", "version": "2026-05-26", "publicationDate": null, "datePrecision": "unknown", "type": "safety-preparedness-report", "verifiedOn": "2026-10-25", "documentId": "https://ai.meta.com/static-resource/muse-spark-safety-and-preparedness-report" }, { "id": "official-meta-muse-wonder-model-card", "title": "Muse Wonder Model Card", "url": "https://research.meta.ai/static/muse-wonder-model-card", "version": "2026-10-23", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://research.meta.ai/static/muse-wonder-model-card" }, { "id": "official-meta-the-llama-3-herd-of-models", "title": "The Llama 3 Herd of Models", "url": "https://arxiv.org/html/2407.21783", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2407.21783" }, { "id": "official-meta-muse-spark-safety-preparedness-report", "title": "Muse Spark Safety & Preparedness Report", "url": "https://arxiv.org/html/2606.12429", "version": "2026-08-24", "publicationDate": "2026-05-14", "datePrecision": "day", "type": "safety-preparedness-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2606.12429" }, { "id": "official-meta-introducing-muse-spark-1-1", "title": "Introducing Muse Spark 1.1", "url": "https://research.meta.ai/blog/introducing-muse-spark-meta-model-api", "version": "2026-07-09", "publicationDate": "2026-07-09", "datePrecision": "day", "type": "release-safety-summary", "verifiedOn": "2026-10-25", "documentId": "https://research.meta.ai/blog/introducing-muse-spark-meta-model-api" }, { "id": "official-meta-muse-spark-1-1-evaluation-report", "title": "Muse Spark 1.1 Evaluation Report", "url": "https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report", "version": "2026-07-09", "publicationDate": "2026-07-09", "datePrecision": "day", "type": "evaluation-report", "verifiedOn": "2026-10-25", "documentId": "https://ai.meta.com/static-resource/muse-spark-1-1-evaluation-report" }, { "id": "official-meta-the-multimodal-intelligence-of-muse-spark-1-2", "title": "The Multimodal Intelligence of Muse Spark 1.2", "url": "https://research.meta.ai/blog/multimodal-intelligence-of-muse-spark-1-2", "version": "2026-08-20", "publicationDate": "2026-08-20", "datePrecision": "day", "type": "release-safety-summary", "verifiedOn": "2026-10-25", "documentId": "https://research.meta.ai/blog/multimodal-intelligence-of-muse-spark-1-2" }, { "id": "official-meta-introducing-muse-spark-1-3", "title": "Introducing Muse Spark 1.3", "url": "https://research.meta.ai/blog/introducing-muse-spark-1-3", "version": "2026-09-02", "publicationDate": "2026-09-02", "datePrecision": "day", "type": "release-safety-summary", "verifiedOn": "2026-10-25", "documentId": "https://research.meta.ai/blog/introducing-muse-spark-1-3" }, { "id": "official-meta-muse-wonder-a-frontier-model-for-personal-professional-ai-agents", "title": "Muse Wonder: A Frontier Model For Personal & Professional AI Agents", "url": "https://research.meta.ai/blog/muse-wonder", "version": "2026-10-12", "publicationDate": "2026-10-12", "datePrecision": "day", "type": "release-safety-summary", "verifiedOn": "2026-10-25", "documentId": "https://research.meta.ai/blog/muse-wonder" }, { "id": "official-meta-introducing-muse-spark-open", "title": "Introducing Muse Spark Open", "url": "https://research.meta.ai/blog/muse-spark-open", "version": "2026-10-15", "publicationDate": "2026-10-15", "datePrecision": "day", "type": "release-safety-summary", "verifiedOn": "2026-10-25", "documentId": "https://research.meta.ai/blog/muse-spark-open" }, { "id": "official-mistral-codestral-official-technical-documentation", "title": "Codestral Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/codestral", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/codestral" }, { "id": "official-mistral-devstral-2-official-technical-documentation", "title": "Devstral 2 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/devstral-2", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/devstral-2" }, { "id": "official-mistral-devstral-medium-1-official-technical-documentation", "title": "Devstral Medium 1 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/devstral-medium-1", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/devstral-medium-1" }, { "id": "official-mistral-devstral-small-1-official-technical-documentation", "title": "Devstral Small 1 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/devstral-small-1", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/devstral-small-1" }, { "id": "official-mistral-devstral-small-2-official-technical-documentation", "title": "Devstral Small 2 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/devstral-small-2", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/devstral-small-2" }, { "id": "official-mistral-leanstral-official-technical-documentation", "title": "Leanstral Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/leanstral", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/leanstral" }, { "id": "official-mistral-magistral-medium-1-official-technical-documentation", "title": "Magistral Medium 1 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/magistral-medium-1", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/magistral-medium-1" }, { "id": "official-mistral-magistral-small-1-official-technical-documentation", "title": "Magistral Small 1 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/magistral-small-1", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/magistral-small-1" }, { "id": "official-mistral-magistral-small-2506-model-card", "title": "Magistral-Small-2506 Model Card", "url": "https://huggingface.co/mistralai/Magistral-Small-2506", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/mistralai/Magistral-Small-2506" }, { "id": "official-mistral-mathstral-7b-official-technical-documentation", "title": "Mathstral 7B Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mathstral-7b", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mathstral-7b" }, { "id": "official-mistral-ministral-1-3b-official-technical-documentation", "title": "Ministral 1 - 3B Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/ministral-1-3b", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/ministral-1-3b" }, { "id": "official-mistral-ministral-1-8b-official-technical-documentation", "title": "Ministral 1 - 8B Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/ministral-1-8b", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/ministral-1-8b" }, { "id": "official-mistral-ministral-3-14b-official-technical-documentation", "title": "Ministral 3 - 14B Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/ministral-3-14b", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/ministral-3-14b" }, { "id": "official-mistral-ministral-3-3b-official-technical-documentation", "title": "Ministral 3 - 3B Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/ministral-3-3b", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/ministral-3-3b" }, { "id": "official-mistral-ministral-3-8b-official-technical-documentation", "title": "Ministral 3 - 8B Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/ministral-3-8b", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/ministral-3-8b" }, { "id": "official-mistral-mistral-7b", "title": "Mistral 7B", "url": "https://arxiv.org/html/2310.06825", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2310.06825" }, { "id": "official-mistral-mistral-7b-official-technical-documentation", "title": "Mistral 7B Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-7b", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-7b" }, { "id": "official-mistral-mistral-embed-official-technical-documentation", "title": "Mistral Embed Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-embed", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-embed" }, { "id": "official-mistral-mistral-large-1-0-official-technical-documentation", "title": "Mistral Large 1.0 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-large-10", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-large-10" }, { "id": "official-mistral-mistral-large-2-0-official-technical-documentation", "title": "Mistral Large 2.0 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-large-2", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-large-2" }, { "id": "official-mistral-mistral-large-3-official-technical-documentation", "title": "Mistral Large 3 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-large-3", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-large-3" }, { "id": "official-mistral-mistral-large-4-official-technical-documentation", "title": "Mistral Large 4 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-large-4", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-large-4" }, { "id": "official-mistral-mistral-medium-1-official-technical-documentation", "title": "Mistral Medium 1 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-medium-1", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-medium-1" }, { "id": "official-mistral-mistral-medium-3-0-official-technical-documentation", "title": "Mistral Medium 3.0 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-medium-3", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-medium-3" }, { "id": "official-mistral-mistral-moderation-official-technical-documentation", "title": "Mistral Moderation Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-moderation", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-moderation" }, { "id": "official-mistral-mistral-nemo-12b-official-technical-documentation", "title": "Mistral Nemo 12B Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-nemo-12b", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-nemo-12b" }, { "id": "official-mistral-mistral-next-official-technical-documentation", "title": "Mistral Next Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-next", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-next" }, { "id": "official-mistral-mistral-ocr-3-official-technical-documentation", "title": "Mistral OCR 3 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-ocr-3", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-ocr-3" }, { "id": "official-mistral-mistral-ocr-4-official-technical-documentation", "title": "Mistral OCR 4 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-ocr-4", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-ocr-4" }, { "id": "official-mistral-mistral-ocr-official-technical-documentation", "title": "Mistral OCR Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-ocr", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-ocr" }, { "id": "official-mistral-mistral-saba-official-technical-documentation", "title": "Mistral Saba Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-saba", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-saba" }, { "id": "official-mistral-mistral-small-1-0-official-technical-documentation", "title": "Mistral Small 1.0 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-small-1", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-small-1" }, { "id": "official-mistral-mistral-small-2-0-official-technical-documentation", "title": "Mistral Small 2.0 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-small-2", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-small-2" }, { "id": "official-mistral-mistral-small-3-0-official-technical-documentation", "title": "Mistral Small 3.0 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-small-3", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-small-3" }, { "id": "official-mistral-mistral-small-4-official-technical-documentation", "title": "Mistral Small 4 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mistral-small-4", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mistral-small-4" }, { "id": "official-mistral-mistral-7b-instruct-v0-1-model-card", "title": "Mistral-7B-Instruct-v0.1 Model Card", "url": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/mistralai/Mistral-7B-Instruct-v0.1" }, { "id": "official-mistral-mistral-large-3-675b-instruct-2512-model-card", "title": "Mistral-Large-3-675B-Instruct-2512 Model Card", "url": "https://huggingface.co/mistralai/Mistral-Large-3-675B-Instruct-2512", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/mistralai/Mistral-Large-3-675B-Instruct-2512" }, { "id": "official-mistral-mistral-large-instruct-2407-model-card", "title": "Mistral-Large-Instruct-2407 Model Card", "url": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2407" }, { "id": "official-mistral-mistral-large-instruct-2411-model-card", "title": "Mistral-Large-Instruct-2411 Model Card", "url": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2411", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/mistralai/Mistral-Large-Instruct-2411" }, { "id": "official-mistral-mistral-small-4-119b-2603-model-card", "title": "Mistral-Small-4-119B-2603 Model Card", "url": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/mistralai/Mistral-Small-4-119B-2603" }, { "id": "official-mistral-mixtral-8b-x-7b-official-technical-documentation", "title": "Mixtral 8B x 7B Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mixtral-8b-x-7b", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mixtral-8b-x-7b" }, { "id": "official-mistral-mixtral-8x22b-official-technical-documentation", "title": "Mixtral 8x22B Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/mixtral-8x22b", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/mixtral-8x22b" }, { "id": "official-mistral-mixtral-8x22b-instruct-v0-1-model-card", "title": "Mixtral-8x22B-Instruct-v0.1 Model Card", "url": "https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/mistralai/Mixtral-8x22B-Instruct-v0.1" }, { "id": "official-mistral-mixtral-8x7b-instruct-v0-1-model-card", "title": "Mixtral-8x7B-Instruct-v0.1 Model Card", "url": "https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/mistralai/Mixtral-8x7B-Instruct-v0.1" }, { "id": "official-mistral-pixtral-12b-official-technical-documentation", "title": "Pixtral 12B Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/pixtral-12b", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/pixtral-12b" }, { "id": "official-mistral-pixtral-large-official-technical-documentation", "title": "Pixtral Large Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/pixtral-large", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/pixtral-large" }, { "id": "official-mistral-robostral-navigate-official-technical-documentation", "title": "Robostral Navigate Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/robostral-navigate", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/robostral-navigate" }, { "id": "official-mistral-shieldstral", "title": "Shieldstral", "url": "https://arxiv.org/html/2607.25857", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2607.25857" }, { "id": "official-mistral-shieldstral-official-technical-documentation", "title": "Shieldstral Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/shieldstral", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/shieldstral" }, { "id": "official-mistral-shieldstral-1-0-3b-model-card", "title": "Shieldstral-1.0-3B Model Card", "url": "https://huggingface.co/mistralai/Shieldstral-1.0-3B", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/mistralai/Shieldstral-1.0-3B" }, { "id": "official-mistral-timestral-official-technical-documentation", "title": "Timestral Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/timestral", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/timestral" }, { "id": "official-mistral-voxtral-2-official-technical-documentation", "title": "Voxtral 2 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/voxtral-2", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/voxtral-2" }, { "id": "official-mistral-voxtral-3-official-technical-documentation", "title": "Voxtral 3 Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/voxtral-3", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/voxtral-3" }, { "id": "official-mistral-voxtral-mini-tts-official-technical-documentation", "title": "Voxtral Mini TTS Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/voxtral-mini-tts", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/voxtral-mini-tts" }, { "id": "official-mistral-voxtral-small-official-technical-documentation", "title": "Voxtral Small Official Technical Documentation", "url": "https://legal.mistral.ai/ai-governance/models/voxtral-small", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-lifecycle-page", "verifiedOn": "2026-10-25", "documentId": "https://legal.mistral.ai/ai-governance/models/voxtral-small" }, { "id": "official-moonshot-kimi-k2-technical-report", "title": "Kimi K2 Technical Report", "url": "https://raw.githubusercontent.com/MoonshotAI/Kimi-K2/main/tech_report.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "https://raw.githubusercontent.com/MoonshotAI/Kimi-K2/main/tech_report.pdf" }, { "id": "official-moonshot-kimi-k2-5-technical-report", "title": "Kimi K2.5 Technical Report", "url": "https://arxiv.org/html/2602.02276", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2602.02276" }, { "id": "official-moonshot-kimi-k3-technical-report", "title": "Kimi K3 Technical Report", "url": "https://raw.githubusercontent.com/MoonshotAI/Kimi-K3/main/k3_tech_report.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "https://raw.githubusercontent.com/MoonshotAI/Kimi-K3/main/k3_tech_report.pdf" }, { "id": "official-moonshot-kimi-k1-5-scaling-reinforcement-learning-with-llms", "title": "Kimi k1.5: Scaling Reinforcement Learning with LLMs", "url": "https://arxiv.org/html/2501.12599", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2501.12599" }, { "id": "official-moonshot-kimi-k2-instruct-model-card", "title": "Kimi-K2-Instruct Model Card", "url": "https://huggingface.co/moonshotai/Kimi-K2-Instruct", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/moonshotai/Kimi-K2-Instruct" }, { "id": "official-moonshot-kimi-k2-instruct-0905-model-card", "title": "Kimi-K2-Instruct-0905 Model Card", "url": "https://huggingface.co/moonshotai/Kimi-K2-Instruct-0905", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/moonshotai/Kimi-K2-Instruct-0905" }, { "id": "official-moonshot-kimi-k2-thinking-model-card", "title": "Kimi-K2-Thinking Model Card", "url": "https://huggingface.co/moonshotai/Kimi-K2-Thinking", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/moonshotai/Kimi-K2-Thinking" }, { "id": "official-moonshot-kimi-k2-5-model-card", "title": "Kimi-K2.5 Model Card", "url": "https://huggingface.co/moonshotai/Kimi-K2.5", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/moonshotai/Kimi-K2.5" }, { "id": "official-moonshot-kimi-k3-model-card", "title": "Kimi-K3 Model Card", "url": "https://huggingface.co/moonshotai/Kimi-K3", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/moonshotai/Kimi-K3" }, { "id": "official-openai-dall-e-3-system-card-pdf-original", "title": "DALL-E 3 System Card", "url": "https://cdn.openai.com/papers/DALL_E_3_System_Card.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://cdn.openai.com/papers/DALL_E_3_System_Card.pdf" }, { "id": "official-openai-dall-e-2-preview-risks-and-limitations", "title": "DALL·E 2 Preview - Risks and Limitations", "url": "https://raw.githubusercontent.com/openai/dalle-2-preview/main/system-card.md", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://raw.githubusercontent.com/openai/dalle-2-preview/main/system-card.md" }, { "id": "official-openai-gpt-4-system-card-pdf-original", "title": "GPT-4 System Card", "url": "https://cdn.openai.com/papers/gpt-4-system-card.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://cdn.openai.com/papers/gpt-4-system-card.pdf" }, { "id": "official-openai-gpt-4-5-system-card-february-original", "title": "GPT-4.5 System Card February Original", "url": "https://cdn.openai.com/gpt-4-5-system-card-2272025.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://cdn.openai.com/gpt-4-5-system-card-2272025.pdf" }, { "id": "official-openai-gpt-4v-system-card-pdf-original", "title": "GPT-4V System Card PDF Original", "url": "https://cdn.openai.com/papers/GPTV_System_Card.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://cdn.openai.com/papers/GPTV_System_Card.pdf" }, { "id": "official-openai-gpt-4o-system-card-pdf-original", "title": "GPT-4o System Card", "url": "https://cdn.openai.com/gpt-4o-system-card.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://cdn.openai.com/gpt-4o-system-card.pdf" }, { "id": "official-openai-openai-o1-preview-system-card-pdf-original", "title": "OpenAI o1 Preview System Card PDF Original", "url": "https://cdn.openai.com/o1-system-card.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://cdn.openai.com/o1-system-card.pdf" }, { "id": "official-openai-openai-o1-system-card-pdf-original", "title": "OpenAI o1 System Card", "url": "https://cdn.openai.com/o1-system-card-20241205.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://cdn.openai.com/o1-system-card-20241205.pdf" }, { "id": "official-openai-openai-o3-and-o4-mini-system-card-pdf-original", "title": "OpenAI o3 and o4-mini System Card", "url": "https://cdn.openai.com/pdf/2221c875-02dc-4789-800b-e7758f3722c1/o3-and-o4-mini-system-card.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://cdn.openai.com/pdf/2221c875-02dc-4789-800b-e7758f3722c1/o3-and-o4-mini-system-card.pdf" }, { "id": "official-openai-openai-o3-mini-system-card-february-revision", "title": "OpenAI o3-mini System Card February Revision", "url": "https://cdn.openai.com/o3-mini-system-card-feb10.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://cdn.openai.com/o3-mini-system-card-feb10.pdf" }, { "id": "official-openai-operator-system-card-pdf-original", "title": "Operator System Card", "url": "https://cdn.openai.com/operator_system_card.pdf", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://cdn.openai.com/operator_system_card.pdf" }, { "id": "official-openai-training-language-models-to-follow-instructions-with-human-feedback", "title": "Training language models to follow instructions with human feedback", "url": "https://arxiv.org/html/2203.02155", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2203.02155" }, { "id": "official-openai-deep-research-system-card", "title": "Deep Research System Card", "url": "https://deploymentsafety.openai.com/deep-research", "version": "2025-02-25", "publicationDate": "2025-02-25", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/deep-research" }, { "id": "official-openai-addendum-to-gpt-4o-system-card-native-image-generation", "title": "Addendum to GPT-4o System Card: Native image generation", "url": "https://deploymentsafety.openai.com/4o-native-image-generation", "version": "2025-03-25", "publicationDate": "2025-03-25", "datePrecision": "day", "type": "system-card-addendum", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/4o-native-image-generation" }, { "id": "official-openai-chatgpt-agent-system-card", "title": "ChatGPT Agent System Card", "url": "https://deploymentsafety.openai.com/chatgpt-agent", "version": "2025-07-17", "publicationDate": "2025-07-17", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/chatgpt-agent" }, { "id": "official-openai-gpt-oss-120b-gpt-oss-20b-model-card", "title": "gpt-oss-120b & gpt-oss-20b Model Card", "url": "https://deploymentsafety.openai.com/gpt-oss", "version": "2025-08-05", "publicationDate": "2025-08-05", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-oss" }, { "id": "official-openai-gpt-5-system-card", "title": "GPT-5 System Card", "url": "https://deploymentsafety.openai.com/gpt-5", "version": "2025-08-07", "publicationDate": "2025-08-07", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-5" }, { "id": "official-openai-sora-2-system-card", "title": "Sora 2 System Card", "url": "https://deploymentsafety.openai.com/sora-2", "version": "2025-09-30", "publicationDate": "2025-09-30", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/sora-2" }, { "id": "official-openai-addendum-to-gpt-5-system-card-sensitive-conversations", "title": "Addendum to GPT-5 System Card: Sensitive Conversations", "url": "https://deploymentsafety.openai.com/gpt-5-sensitive-conversations", "version": "2025-10-27", "publicationDate": "2025-10-27", "datePrecision": "day", "type": "system-card-addendum", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-5-sensitive-conversations" }, { "id": "official-openai-gpt-5-1-instant-and-gpt-5-1-thinking-system-card-addendum", "title": "GPT-5.1 Instant and GPT-5.1 Thinking System Card Addendum", "url": "https://deploymentsafety.openai.com/gpt-5-1", "version": "2025-11-12", "publicationDate": "2025-11-12", "datePrecision": "day", "type": "system-card-addendum", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-5-1" }, { "id": "official-openai-gpt-5-1-codex-max-system-card", "title": "GPT-5.1-Codex-Max System Card", "url": "https://deploymentsafety.openai.com/gpt-5-1-codex-max", "version": "2025-11-18", "publicationDate": "2025-11-18", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-5-1-codex-max" }, { "id": "official-openai-update-to-gpt-5-system-card-gpt-5-2", "title": "Update to GPT-5 System Card: GPT-5.2", "url": "https://deploymentsafety.openai.com/gpt-5-2", "version": "2025-12-11", "publicationDate": "2025-12-11", "datePrecision": "day", "type": "system-card-addendum", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-5-2" }, { "id": "official-openai-addendum-to-gpt-5-2-system-card-gpt-5-2-codex", "title": "Addendum to GPT-5.2 System Card: GPT-5.2-Codex", "url": "https://deploymentsafety.openai.com/gpt-5-2-codex", "version": "2025-12-18", "publicationDate": "2025-12-18", "datePrecision": "day", "type": "system-card-addendum", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-5-2-codex" }, { "id": "official-openai-gpt-5-3-codex-system-card", "title": "GPT-5.3-Codex System Card", "url": "https://deploymentsafety.openai.com/gpt-5-3-codex", "version": "2026-02-05", "publicationDate": "2026-02-05", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-5-3-codex" }, { "id": "official-openai-gpt-5-3-instant-system-card", "title": "GPT-5.3 Instant System Card", "url": "https://deploymentsafety.openai.com/gpt-5-3-instant", "version": "2026-03-02", "publicationDate": "2026-03-02", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-5-3-instant" }, { "id": "official-openai-gpt-5-4-thinking-system-card", "title": "GPT-5.4 Thinking System Card", "url": "https://deploymentsafety.openai.com/gpt-5-4-thinking", "version": "2026-03-05", "publicationDate": "2026-03-05", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-5-4-thinking" }, { "id": "official-openai-chatgpt-images-2-0-system-card", "title": "ChatGPT Images 2.0 System Card", "url": "https://deploymentsafety.openai.com/chatgpt-images-2-0", "version": "2026-04-21", "publicationDate": "2026-04-21", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/chatgpt-images-2-0" }, { "id": "official-openai-gpt-5-5-system-card", "title": "GPT-5.5 System Card", "url": "https://deploymentsafety.openai.com/gpt-5-5", "version": "2026-04-23", "publicationDate": "2026-04-23", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-5-5" }, { "id": "official-openai-gpt-5-5-instant-system-card", "title": "GPT-5.5 Instant System Card", "url": "https://deploymentsafety.openai.com/gpt-5-5-instant", "version": "2026-05-05", "publicationDate": "2026-05-05", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-5-5-instant" }, { "id": "official-openai-gpt-rosalind-5-5-system-card", "title": "GPT-Rosalind-5.5 System Card", "url": "https://deploymentsafety.openai.com/gpt-rosalind-5-5", "version": "2026-06-03", "publicationDate": "2026-06-03", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-rosalind-5-5" }, { "id": "official-openai-gpt-5-6-preview-system-card", "title": "GPT-5.6 Preview System Card", "url": "https://deploymentsafety.openai.com/gpt-5-6-preview", "version": "2026-06-26", "publicationDate": "2026-06-26", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-5-6-preview" }, { "id": "official-openai-gpt-live-system-card", "title": "GPT-Live System Card", "url": "https://deploymentsafety.openai.com/gpt-live", "version": "2026-07-08", "publicationDate": "2026-07-08", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-live" }, { "id": "official-openai-gpt-5-6-system-card", "title": "GPT-5.6 System Card", "url": "https://deploymentsafety.openai.com/gpt-5-6", "version": "2026-07-09", "publicationDate": "2026-07-09", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-5-6" }, { "id": "official-openai-gpt-5-6-august-updates", "title": "GPT-5.6 — August Updates", "url": "https://deploymentsafety.openai.com/gpt-5-6-august-update", "version": "2026-08-06", "publicationDate": "2026-08-06", "datePrecision": "day", "type": "system-card-addendum", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-5-6-august-update" }, { "id": "official-openai-gpt-6-astra-system-card", "title": "GPT-6 Astra System Card", "url": "https://deploymentsafety.openai.com/gpt-6-astra", "version": "2026-09-09", "publicationDate": "2026-09-03", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-6-astra" }, { "id": "official-openai-chatgpt-images-2-5-system-card", "title": "ChatGPT Images 2.5 System Card", "url": "https://deploymentsafety.openai.com/chatgpt-images-2-5", "version": "2026-09-08", "publicationDate": "2026-09-08", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/chatgpt-images-2-5" }, { "id": "official-openai-addendum-to-gpt-6-astra-system-card-gpt-6-1-sol", "title": "Addendum to GPT-6 Astra System Card: GPT-6.1 Sol", "url": "https://deploymentsafety.openai.com/gpt-6-1-sol", "version": "2026-09-29", "publicationDate": "2026-09-29", "datePrecision": "day", "type": "system-card-addendum", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-6-1-sol" }, { "id": "official-openai-gpt-6-sol-and-gpt-6-luna-october-2026-update", "title": "GPT-6 Sol and GPT-6 Luna: October 2026 update", "url": "https://deploymentsafety.openai.com/gpt-6-october", "version": "2026-10-07", "publicationDate": "2026-10-07", "datePrecision": "day", "type": "system-card-addendum", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-6-october" }, { "id": "official-openai-gpt-live-1-1-system-card", "title": "GPT-Live-1.1 system card", "url": "https://deploymentsafety.openai.com/gpt-live-1-1", "version": "2026-10-15", "publicationDate": "2026-10-15", "datePrecision": "day", "type": "system-card", "verifiedOn": "2026-10-25", "documentId": "https://deploymentsafety.openai.com/gpt-live-1-1" }, { "id": "official-qwen-qwen-technical-report", "title": "Qwen Technical Report", "url": "https://arxiv.org/html/2309.16609", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2309.16609" }, { "id": "official-qwen-qwen-72b-chat-model-card", "title": "Qwen-72B-Chat Model Card", "url": "https://huggingface.co/Qwen/Qwen-72B-Chat", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/Qwen/Qwen-72B-Chat" }, { "id": "official-qwen-qwen2-technical-report", "title": "Qwen2 Technical Report", "url": "https://arxiv.org/html/2407.10671", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2407.10671" }, { "id": "official-qwen-qwen2-72b-instruct-model-card", "title": "Qwen2-72B-Instruct Model Card", "url": "https://huggingface.co/Qwen/Qwen2-72B-Instruct", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/Qwen/Qwen2-72B-Instruct" }, { "id": "official-qwen-qwen2-5-72b-instruct-model-card", "title": "Qwen2.5-72B-Instruct Model Card", "url": "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/Qwen/Qwen2.5-72B-Instruct" }, { "id": "official-qwen-qwen3-235b-a22b-model-card", "title": "Qwen3-235B-A22B Model Card", "url": "https://huggingface.co/Qwen/Qwen3-235B-A22B", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/Qwen/Qwen3-235B-A22B" }, { "id": "official-qwen-qwen3-5-397b-a17b-model-card", "title": "Qwen3.5-397B-A17B Model Card", "url": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/Qwen/Qwen3.5-397B-A17B" }, { "id": "official-qwen-qwen3-8-2-4t-a95b-model-card", "title": "Qwen3.8-2.4T-A95B Model Card", "url": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/Qwen/Qwen3.8-2.4T-A95B" }, { "id": "official-qwen-qwen3-8-flash-next-model-card", "title": "Qwen3.8-Flash-Next Model Card", "url": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/Qwen/Qwen3.8-Flash-Next" }, { "id": "official-qwen-qwen3guard-technical-report", "title": "Qwen3Guard Technical Report", "url": "https://arxiv.org/html/2510.14276", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2510.14276" }, { "id": "official-qwen-qwen3guard-gen-8b-model-card", "title": "Qwen3Guard-Gen-8B Model Card", "url": "https://huggingface.co/Qwen/Qwen3Guard-Gen-8B", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/Qwen/Qwen3Guard-Gen-8B" }, { "id": "official-qwen-qwen3guard-stream-8b-model-card", "title": "Qwen3Guard-Stream-8B Model Card", "url": "https://huggingface.co/Qwen/Qwen3Guard-Stream-8B", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/Qwen/Qwen3Guard-Stream-8B" }, { "id": "official-qwen-qwen3-technical-report", "title": "Qwen3 Technical Report", "url": "https://arxiv.org/html/2505.09388", "version": "2025-05-14", "publicationDate": "2025-05-14", "datePrecision": "day", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2505.09388" }, { "id": "official-qwen-qwen3-8-flash-next-technical-report", "title": "On the Design of Qwen3.8-Next Architecture: Evaluation, Efficiency, and Training Stability", "url": "https://raw.githubusercontent.com/QwenLM/Qwen3.8-Flash-Next/main/tech_report.pdf", "version": "2026-08-26", "publicationDate": "2026-08-26", "datePrecision": "day", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "https://raw.githubusercontent.com/QwenLM/Qwen3.8-Flash-Next/main/tech_report.pdf" }, { "id": "official-xai-grok-4-model-card", "title": "Grok 4 Model Card", "url": "https://data.x.ai/2025-08-20-grok-4-model-card.pdf", "version": "2025-08-20", "publicationDate": "2025-08-20", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://data.x.ai/2025-08-20-grok-4-model-card.pdf" }, { "id": "official-xai-grok-code-fast-1-model-card", "title": "Grok Code Fast 1 Model Card", "url": "https://data.x.ai/2025-08-26-grok-code-fast-1-model-card.pdf", "version": "2025-08-26", "publicationDate": "2025-08-26", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://data.x.ai/2025-08-26-grok-code-fast-1-model-card.pdf" }, { "id": "official-xai-grok-4-fast-model-card", "title": "Grok 4 Fast Model Card", "url": "https://data.x.ai/2025-09-19-grok-4-fast-model-card.pdf", "version": "2025-09-19", "publicationDate": "2025-09-19", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://data.x.ai/2025-09-19-grok-4-fast-model-card.pdf" }, { "id": "official-xai-grok-4-5-model-card", "title": "Grok 4.5 Model Card", "url": "https://media.x.ai/v1/website/4p5-5184fdf9.pdf", "version": "2026-07-20", "publicationDate": "2026-07-14", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://media.x.ai/v1/website/4p5-5184fdf9.pdf" }, { "id": "official-xai-grok-4-6-model-card", "title": "Grok 4.6 Model Card", "url": "https://media.x.ai/v1/website/card-4p6-4cd2dc57.pdf", "version": "2026-08-17", "publicationDate": "2026-08-12", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://media.x.ai/v1/website/card-4p6-4cd2dc57.pdf" }, { "id": "official-xai-grok-4-7-model-card", "title": "Grok 4.7 Model Card", "url": "https://media.x.ai/v1/website/card4p7-3a96f40b.pdf", "version": "2026-09-21", "publicationDate": "2026-09-21", "datePrecision": "day", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://media.x.ai/v1/website/card4p7-3a96f40b.pdf" }, { "id": "official-zai-glm-4-5-model-card", "title": "GLM-4.5 Model Card", "url": "https://huggingface.co/zai-org/GLM-4.5", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/zai-org/GLM-4.5" }, { "id": "official-zai-glm-4-5-agentic-reasoning-and-coding-arc-foundation-models", "title": "GLM-4.5: Agentic, Reasoning, and Coding (ARC) Foundation Models", "url": "https://arxiv.org/html/2508.06471", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2508.06471" }, { "id": "official-zai-glm-4-6-model-card", "title": "GLM-4.6 Model Card", "url": "https://huggingface.co/zai-org/GLM-4.6", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/zai-org/GLM-4.6" }, { "id": "official-zai-glm-4-7-model-card", "title": "GLM-4.7 Model Card", "url": "https://huggingface.co/zai-org/GLM-4.7", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/zai-org/GLM-4.7" }, { "id": "official-zai-glm-5-model-card", "title": "GLM-5 Model Card", "url": "https://huggingface.co/zai-org/GLM-5", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/zai-org/GLM-5" }, { "id": "official-zai-glm-5-1-model-card", "title": "GLM-5.1 Model Card", "url": "https://huggingface.co/zai-org/GLM-5.1", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/zai-org/GLM-5.1" }, { "id": "official-zai-glm-5-2-model-card", "title": "GLM-5.2 Model Card", "url": "https://huggingface.co/zai-org/GLM-5.2", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/zai-org/GLM-5.2" }, { "id": "official-zai-glm-5-3-bf16-model-card", "title": "GLM-5.3-BF16 Model Card", "url": "https://huggingface.co/zai-org/GLM-5.3-BF16", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "model-card", "verifiedOn": "2026-10-25", "documentId": "https://huggingface.co/zai-org/GLM-5.3-BF16" }, { "id": "official-zai-glm-5-from-vibe-coding-to-agentic-engineering", "title": "GLM-5: from Vibe Coding to Agentic Engineering", "url": "https://arxiv.org/html/2602.15763", "version": "Official source captured 2026-10-25", "publicationDate": null, "datePrecision": "unknown", "type": "technical-report", "verifiedOn": "2026-10-25", "documentId": "arxiv:2602.15763" } ], "families": [ { "id": "agentic-risk", "label": { "en": "Agentic Risks", "zh": "主体性风险" }, "description": { "en": "Risk questions about a system acting beyond intended human direction, including persistence and deceptive conduct.", "zh": "研究系统超出预期人类指令采取行动的问题，包括持续行动与欺骗性行为。" } }, { "id": "dual-use-risk", "label": { "en": "Dual-Use Risks", "zh": "两用性风险" }, "description": { "en": "Risk questions about capabilities that can serve legitimate work or enable harm under different actors and conditions.", "zh": "研究同一能力在不同主体与条件下既可用于正当任务、也可能促成危害的问题。" } } ], "areas": [ { "id": "autonomy", "family": "agentic-risk", "label": { "en": "Autonomy", "zh": "自主性" }, "definition": { "en": "When can an agent pursue consequential actions beyond intended direction or resist intervention?", "zh": "智能体在何种条件下可能超出预期指令持续采取有影响的行动，或抵抗干预？" }, "scope": { "en": "Sustained task execution, replication, and resistance to interruption; capability alone is not unauthorized action.", "zh": "持续任务执行、复制及对中断的抵抗；具备能力不等于未经授权行动。" }, "threatModel": { "en": "A system with tools and access may continue or extend action despite a conflicting instruction or intervention.", "zh": "具备工具及访问权限的系统可能在相反指令或干预下继续或扩展行动。" }, "gap": { "en": "Controlled tasks do not establish real-world proliferation or loss of control.", "zh": "受控任务不证明现实中的自主扩散或失控。" } }, { "id": "deception", "family": "agentic-risk", "label": { "en": "Deception", "zh": "欺骗" }, "definition": { "en": "When does a system conceal, misrepresent, or strategically alter behavior toward users or overseers?", "zh": "系统何时会对用户或监督者隐瞒、误导或策略性改变行为？" }, "scope": { "en": "Scheming, evaluation gaming, and misleading communication are distinct questions.", "zh": "隐蔽谋划、评测规避与误导性交流是不同问题。" }, "threatModel": { "en": "An agent may have a stated objective and opportunities to hide actions or alter reports.", "zh": "智能体可能具有给定目标，并有机会隐藏行动或改变报告。" }, "gap": { "en": "Incorrect answers and evaluation awareness alone do not demonstrate intentional deception.", "zh": "错误回答或意识到正在评测，本身不能证明有意欺骗。" } }, { "id": "cybersecurity", "family": "dual-use-risk", "label": { "en": "Cybersecurity", "zh": "网络安全" }, "definition": { "en": "How might AI systems perform or enable cyber operations, and how can agents be compromised?", "zh": "AI 系统如何执行或促成网络行动，智能体又如何受到攻击？" }, "scope": { "en": "Offensive task capability and agent compromise are tracked separately; authorized security research is not observed criminal use.", "zh": "分别记录攻击任务能力与智能体受侵害；经授权的安全研究不是犯罪使用观测。" }, "threatModel": { "en": "A specified actor uses an AI system, access, tools, and targets in a cyber task.", "zh": "特定主体在网络任务中使用 AI 系统、访问权限、工具与目标。" }, "gap": { "en": "Benchmark success cannot establish real-world attack prevalence or expected losses.", "zh": "基准成功率不能推断真实攻击发生率或预期损失。" } }, { "id": "cbrn", "family": "dual-use-risk", "label": { "en": "CBRN", "zh": "化生放核" }, "definition": { "en": "What assistance could AI provide across hazardous scientific knowledge and consequential task steps?", "zh": "AI 在危险科学知识及关键任务环节可能提供何种协助？" }, "scope": { "en": "Track knowledge probes separately from practical task uplift; publish only bounded, non-operational summaries.", "zh": "区分知识测验与实际任务增益；只公开有边界的非操作性摘要。" }, "threatModel": { "en": "A user with defined expertise and resources seeks help for a hazardous task.", "zh": "具备特定专业知识及资源的使用者寻求危险任务协助。" }, "gap": { "en": "Multiple-choice knowledge does not prove end-to-end harmful capability.", "zh": "选择题知识成绩不证明端到端危害能力。" } }, { "id": "disinformation", "family": "dual-use-risk", "label": { "en": "Disinformation", "zh": "虚假信息与影响" }, "definition": { "en": "When can AI-generated content or interaction influence beliefs at scale or support deceptive campaigns?", "zh": "AI 生成内容或交互何时可能规模化影响信念或支持误导性活动？" }, "scope": { "en": "Persuasion, synthetic content, and false claims have different measurement requirements.", "zh": "说服效果、合成内容与虚假陈述需要不同的测量方式。" }, "threatModel": { "en": "An actor uses model output or interaction to influence an audience under defined conditions.", "zh": "行为主体在明确条件下使用模型输出或交互影响受众。" }, "gap": { "en": "Persuasion effects do not by themselves prove false information or societal-scale impact.", "zh": "说服效果本身不证明内容虚假或具有社会规模影响。" } }, { "id": "autonomous-physical-harm", "family": "dual-use-risk", "label": { "en": "Autonomous physical harm", "zh": "自主物理危害" }, "definition": { "en": "What risks arise when AI systems perceive, decide, and act in physical environments?", "zh": "AI 系统在物理环境中感知、决策并行动时会产生哪些风险？" }, "scope": { "en": "Distinguish hazard recognition from real embodied action and control.", "zh": "区分危险识别与真实具身行动及控制。" }, "threatModel": { "en": "An agent has sensors, actuators, task instructions, and opportunities for physical action.", "zh": "智能体具备传感器、执行器、任务指令及物理行动机会。" }, "gap": { "en": "Image-based safety reasoning is not evidence of reliable robotic operation or harm.", "zh": "图像安全推理不能证明可靠机器人操作或实际危害。" } } ], "benchmarks": [ { "id": "agentcyberrange", "name": "AgentCyberRange", "kind": "benchmark", "sourceId": "agentcyberrange", "version": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "summary": { "en": "Autonomous cyber tasks in enterprise-like ranges.", "zh": "企业式靶场中的自主网络任务。" }, "measures": { "en": "Task success across web exploitation and post-exploitation settings; configurations must be checked per submission.", "zh": "Web 利用与后渗透任务表现；每项提交分别核查配置。" }, "limitations": { "en": "Changing task sets and submitted agent configurations prevent unrestricted comparison.", "zh": "任务集与提交配置变化，不支持无条件比较。" }, "tags": [ "published-evaluation" ], "primaryDimension": "capability", "assessmentSubject": { "en": "AI agent completing cyber-range tasks", "zh": "AI 智能体完成靶场任务" }, "links": [ { "questionId": "cyber-task-execution", "dimension": "capability", "relation": "direct", "subject": { "en": "AI agent completing cyber-range tasks", "zh": "AI 智能体完成靶场任务" }, "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "Official leaderboard, Level-0 tracks" } ], "classificationNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" } }, { "id": "asimov", "name": "ASIMOV v2", "kind": "benchmark", "sourceId": "asimov", "version": "ASIMOV v2 / arXiv:2509.21651", "summary": { "en": "Perception of physical danger and safety interventions.", "zh": "物理危险感知与安全干预。" }, "measures": { "en": "Multimodal safety reasoning on generated scenes.", "zh": "生成场景上的多模态安全推理。" }, "limitations": { "en": "Scene-based reasoning does not certify a deployed robot's physical safety.", "zh": "场景推理不构成部署机器人物理安全认证。" }, "tags": [ "embodied" ], "primaryDimension": "capability", "assessmentSubject": { "en": "Model interpreting multimodal physical-risk scenes", "zh": "解读多模态物理风险场景的模型" }, "links": [ { "questionId": "physical-hazard-recognition", "dimension": "capability", "relation": "direct", "subject": { "en": "Model interpreting multimodal physical-risk scenes", "zh": "解读多模态物理风险场景的模型" }, "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "ASIMOV v2, scene reasoning tasks" } ], "classificationNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" } }, { "id": "cybench", "name": "Cybench", "kind": "benchmark", "sourceId": "cybench", "version": "arXiv:2408.08926v4", "summary": { "en": "Cyber capability on competition tasks with optional subtask guidance.", "zh": "竞赛任务及可选子任务提示下的网络能力。" }, "measures": { "en": "Task completion on 40 professional CTF problems.", "zh": "40 项专业 CTF 任务的完成情况。" }, "limitations": { "en": "Competition tasks and supplied guidance differ from operational attacks.", "zh": "竞赛任务与提供的提示不同于现实攻击条件。" }, "tags": [ "published-evaluation" ], "primaryDimension": "capability", "assessmentSubject": { "en": "AI agent solving CTF tasks", "zh": "完成 CTF 任务的 AI 智能体" }, "links": [ { "questionId": "cyber-task-execution", "dimension": "capability", "relation": "direct", "subject": { "en": "AI agent solving CTF tasks", "zh": "完成 CTF 任务的 AI 智能体" }, "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "Paper abstract and task protocol" } ], "classificationNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" } }, { "id": "cybergym", "name": "CyberGym", "kind": "benchmark", "sourceId": "cybergym", "version": "arXiv:2506.02548v3", "summary": { "en": "Vulnerability reproduction in real software projects.", "zh": "真实软件项目中的漏洞复现。" }, "measures": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "limitations": { "en": "Vulnerability reproduction is narrower than autonomous compromise of deployed systems.", "zh": "漏洞复现的范围小于对部署系统的自主入侵。" }, "tags": [ "published-evaluation" ], "primaryDimension": "capability", "assessmentSubject": { "en": "AI agent reproducing historical vulnerabilities", "zh": "复现历史漏洞的 AI 智能体" }, "links": [ { "questionId": "vulnerability-reproduction", "dimension": "capability", "relation": "direct", "subject": { "en": "AI agent reproducing historical vulnerabilities", "zh": "复现历史漏洞的 AI 智能体" }, "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "Paper abstract and validated trigger protocol" } ], "classificationNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" } }, { "id": "in-context-scheming", "name": "In-context scheming", "kind": "study", "sourceId": "in-context-scheming", "version": "arXiv:2412.04984v2", "summary": { "en": "Covert goal pursuit in environments that incentivize scheming.", "zh": "激励隐蔽谋划的环境中的秘密目标追求。" }, "measures": { "en": "Behavior under supplied goals and evaluation-specific incentives.", "zh": "给定目标和评测特定激励下的行为。" }, "limitations": { "en": "Goal provision and experimental incentives limit propensity claims.", "zh": "给定目标与实验激励限制了行为倾向结论。" }, "tags": [ "published-evaluation" ], "primaryDimension": "capability", "assessmentSubject": { "en": "Model assigned a conflicting goal in context", "zh": "上下文中被赋予冲突目标的模型" }, "links": [ { "questionId": "covert-goal-pursuit", "dimension": "capability", "relation": "direct", "subject": { "en": "Model assigned a conflicting goal in context", "zh": "上下文中被赋予冲突目标的模型" }, "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "arXiv:2412.04984v2, goal-conflict evaluations" }, { "questionId": "covert-goal-pursuit", "dimension": "behavior", "relation": "direct", "subject": { "en": "Model assigned a conflicting goal in context", "zh": "上下文中被赋予冲突目标的模型" }, "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "Reported actions within controlled scenarios" } ], "classificationNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" } }, { "id": "aisi-cyber-scenarios", "name": "Measuring AI Agents’ Progress on Multi-Step Cyber Attack Scenarios", "kind": "study", "primaryDimension": "capability", "assessmentSubject": { "en": "Agents in two purpose-built cyber ranges", "zh": "两套专设网络靶场中的智能体" }, "links": [ { "questionId": "cyber-task-execution", "dimension": "capability", "relation": "direct", "subject": { "en": "AI agent systems", "zh": "AI 智能体系统" }, "rationale": { "en": "Task completion in a bounded range measures capability under its specific protocol.", "zh": "有限靶场的任务完成情况只测量特定协议下的能力。" }, "locator": "Research abstract; full technical paper linked" } ], "sourceId": "aisi-cyber-range", "version": "Research page checked 2026-10-25", "summary": { "en": "Assesses autonomous progress through multi-step cyber scenarios.", "zh": "评估智能体在多步网络场景中的自主推进。" }, "measures": { "en": "Task progress in two defined ranges under varied inference budgets.", "zh": "在两套明确靶场及不同推理预算下的任务推进。" }, "limitations": { "en": "A purpose-built study is not an open-ended estimate of real-world cyber attack success.", "zh": "专设研究不代表现实网络攻击成功概率。" }, "tags": [ "source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Primary dimension follows task completion, with budget and scenario constraints.", "zh": "主维度依据任务完成情况，须同时呈现预算及场景约束。" } }, { "id": "time-horizons", "name": "METR Time Horizons", "kind": "benchmark", "sourceId": "time-horizons", "version": "Time Horizon 1.1; page updated 2026-09-08", "summary": { "en": "Task difficulty expressed in human expert completion time.", "zh": "以人类专家完成时间表示的任务难度。" }, "measures": { "en": "Fitted success probability across a software-task distribution.", "zh": "软件任务分布上的拟合成功概率。" }, "limitations": { "en": "The public page is no longer actively updated; time horizon is not agent runtime or a universal automation measure.", "zh": "该公开页面已停止持续更新；时间范围不是智能体运行时长或通用自动化指标。" }, "tags": [ "published-evaluation" ], "primaryDimension": "capability", "assessmentSubject": { "en": "AI agent completing software tasks under a harness", "zh": "在脚手架下完成软件任务的 AI 智能体" }, "links": [ { "questionId": "sustained-autonomous-work", "dimension": "capability", "relation": "enabling", "subject": { "en": "AI agent completing software tasks under a harness", "zh": "在脚手架下完成软件任务的 AI 智能体" }, "rationale": { "en": "The source measures a prerequisite capability; it does not show the risk behavior.", "zh": "来源测量前提能力，并未观察到对应风险行为。" }, "locator": "METR time-horizon task distribution" } ], "classificationNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" } }, { "id": "re-bench", "name": "RE-Bench", "kind": "benchmark", "sourceId": "re-bench", "version": "arXiv:2411.15114v2", "summary": { "en": "Research engineering performance under time budgets.", "zh": "时间预算约束下的研究工程表现。" }, "measures": { "en": "ML research engineering outcomes relative to human experts.", "zh": "相对于人类专家的机器学习研究工程成果。" }, "limitations": { "en": "AI R&D performance is an enabling capability, not evidence of harmful self-improvement.", "zh": "AI 研发表现属于促成能力，不构成有害自我改进证据。" }, "tags": [ "enabling-capability", "ai-rd" ], "primaryDimension": "capability", "assessmentSubject": { "en": "AI agent performing ML research engineering tasks", "zh": "执行机器学习研究工程任务的 AI 智能体" }, "links": [ { "questionId": "sustained-autonomous-work", "dimension": "capability", "relation": "enabling", "subject": { "en": "AI agent performing ML research engineering tasks", "zh": "执行机器学习研究工程任务的 AI 智能体" }, "rationale": { "en": "The source measures a prerequisite capability; it does not show the risk behavior.", "zh": "来源测量前提能力，并未观察到对应风险行为。" }, "locator": "arXiv:2411.15114v2, ML R&D tasks" } ], "classificationNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" } }, { "id": "replibench", "name": "RepliBench", "kind": "benchmark", "sourceId": "replibench", "version": "arXiv:2504.18565v2", "summary": { "en": "Capabilities needed for autonomous replication.", "zh": "自主复制所需的能力。" }, "measures": { "en": "Task-family performance on components of replication.", "zh": "复制组成任务上的表现。" }, "limitations": { "en": "Component success does not establish autonomous end-to-end proliferation.", "zh": "组成任务成功不构成端到端自主扩散证据。" }, "tags": [ "published-evaluation" ], "primaryDimension": "capability", "assessmentSubject": { "en": "AI agent completing replication components", "zh": "完成复制组成任务的 AI 智能体" }, "links": [ { "questionId": "self-replication", "dimension": "capability", "relation": "direct", "subject": { "en": "AI agent completing replication components", "zh": "完成复制组成任务的 AI 智能体" }, "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "Task-family definitions in source" } ], "classificationNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" } }, { "id": "self-replication", "name": "Self-Replication", "kind": "study", "sourceId": "self-replication", "version": "arXiv:2412.12140v1", "summary": { "en": "Whether instructed systems create a separate live copy in controlled trials.", "zh": "受指令系统能否在受控试验中建立独立存活副本。" }, "measures": { "en": "Trial outcomes for two model-driven systems.", "zh": "两个模型驱动系统的试验结果。" }, "limitations": { "en": "The supplied environment, explicit instruction and scaffold bound the finding.", "zh": "所提供环境、明确指令与脚手架限定了结论。" }, "tags": [ "published-evaluation" ], "primaryDimension": "capability", "assessmentSubject": { "en": "Prompted model-driven system in a controlled environment", "zh": "受控环境中按指令行动的模型系统" }, "links": [ { "questionId": "self-replication", "dimension": "capability", "relation": "direct", "subject": { "en": "Prompted model-driven system in a controlled environment", "zh": "受控环境中按指令行动的模型系统" }, "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "arXiv:2412.12140v1, controlled trials" } ], "classificationNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" } }, { "id": "truthful-sad", "name": "Situational Awareness Dataset (SAD)", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Language models answering situational-awareness tasks", "zh": "回答情境意识任务的语言模型" }, "links": [ { "questionId": "evaluation-gaming", "dimension": "capability", "relation": "enabling", "subject": { "en": "Language model", "zh": "语言模型" }, "rationale": { "en": "The benchmark measures awareness abilities; it does not test deliberate sandbagging.", "zh": "该基准测量识别能力，不测试有意低表现。" }, "locator": "Benchmark description; full paper linked" } ], "sourceId": "truthful-sad", "version": "2024-07-15 article; original paper linked", "summary": { "en": "A multi-task benchmark for self-knowledge and situational awareness.", "zh": "测量自我知识与情境意识的多任务基准。" }, "measures": { "en": "Task accuracy on whether the model recognizes itself, its context, and possible influence.", "zh": "模型识别自身、所处环境及可影响范围的任务准确率。" }, "limitations": { "en": "Situational awareness is a possible prerequisite for evaluation gaming, not evidence that gaming occurred.", "zh": "情境意识可能是评测规避的前提，不证明规避行为已经发生。" }, "tags": [ "source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Capability is the primary construct. This is enabling evidence, not direct deception evidence.", "zh": "主测量构念为能力；属于前提证据，而非直接欺骗证据。" } }, { "id": "stealth-evals", "name": "Stealth evaluations", "kind": "benchmark", "sourceId": "stealth-evals", "version": "arXiv:2505.01420v4", "summary": { "en": "Reasoning prerequisites for evading oversight.", "zh": "规避监督所需的推理前置能力。" }, "measures": { "en": "Reasoning about oversight and deployment context.", "zh": "有关监督机制与部署情境的推理。" }, "limitations": { "en": "Prerequisite capability is distinct from a tendency to scheme.", "zh": "前置能力不同于实施隐蔽谋划的倾向。" }, "tags": [ "published-evaluation" ], "primaryDimension": "capability", "assessmentSubject": { "en": "Model reasoning about oversight and test context", "zh": "推理监督及测试情境的模型" }, "links": [ { "questionId": "evaluation-gaming", "dimension": "capability", "relation": "enabling", "subject": { "en": "Model reasoning about oversight and test context", "zh": "推理监督及测试情境的模型" }, "rationale": { "en": "The source measures a prerequisite capability; it does not show the risk behavior.", "zh": "来源测量前提能力，并未观察到对应风险行为。" }, "locator": "arXiv:2505.01420v4, stealth and situational awareness tests" } ], "classificationNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" } }, { "id": "wmdp", "name": "WMDP", "kind": "benchmark", "sourceId": "wmdp", "version": "arXiv:2403.03218v7", "summary": { "en": "Hazardous knowledge proxies and unlearning evaluation.", "zh": "危险知识代理测量与遗忘效果评测。" }, "measures": { "en": "Multiple-choice knowledge measurements.", "zh": "选择题形式的知识测量。" }, "limitations": { "en": "Knowledge is a proxy; scores do not establish practical harmful-task ability.", "zh": "知识属于代理指标；分数不能确定实际有害任务能力。" }, "tags": [ "knowledge-proxy" ], "primaryDimension": "capability", "assessmentSubject": { "en": "Model answering hazardous knowledge questions", "zh": "回答危险知识问题的模型" }, "links": [ { "questionId": "hazardous-knowledge", "dimension": "capability", "relation": "direct", "subject": { "en": "Model answering hazardous knowledge questions", "zh": "回答危险知识问题的模型" }, "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "arXiv:2403.03218v7, multiple-choice task design" } ], "classificationNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" } }, { "id": "opendeception", "name": "OpenDeception", "kind": "benchmark", "sourceId": "opendeception", "version": "arXiv:2504.13707v3", "summary": { "en": "Deception in simulated human–AI interactions.", "zh": "模拟人机交互中的欺骗。" }, "measures": { "en": "Scenario-based inferred intent and modeled user trust.", "zh": "基于情境的意图推断与用户信任建模。" }, "limitations": { "en": "Inferred intent is a measurement construct; simulated users do not establish human effects.", "zh": "推断意图是测量构念；模拟用户不能确定真人效果。" }, "tags": [ "published-evaluation" ], "primaryDimension": "propensity", "assessmentSubject": { "en": "Model in simulated social interactions", "zh": "模拟社会交互中的模型" }, "links": [ { "questionId": "misleading-communication", "dimension": "propensity", "relation": "direct", "subject": { "en": "Model in simulated social interactions", "zh": "模拟社会交互中的模型" }, "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "Scenario-level intent measure, arXiv:2504.13707v3" }, { "questionId": "misleading-communication", "dimension": "behavior", "relation": "direct", "subject": { "en": "Model in simulated social interactions", "zh": "模拟社会交互中的模型" }, "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "Simulated interaction outcomes, arXiv:2504.13707v3" } ], "classificationNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" } }, { "id": "aisi-propensity-study", "name": "Propensity Inference: Environmental Contributors to LLM Behaviour", "kind": "study", "primaryDimension": "propensity", "assessmentSubject": { "en": "Language models in controlled evaluation environments", "zh": "受控评测环境中的语言模型" }, "links": [ { "questionId": "unsanctioned-action", "dimension": "propensity", "relation": "direct", "subject": { "en": "Language models", "zh": "语言模型" }, "rationale": { "en": "The study explicitly estimates propensity for unsanctioned behavior under varied environments.", "zh": "研究明确在变化环境下估计未经授权行为倾向。" }, "locator": "Abstract; methods linked from research page" } ], "sourceId": "aisi-propensity", "version": "Research page checked 2026-10-25", "summary": { "en": "Studies how strategic and non-strategic environmental factors contribute to unsanctioned behavior.", "zh": "研究策略性与非策略性环境因素如何影响未经授权行为。" }, "measures": { "en": "Condition-dependent propensity estimates across model and environment variations.", "zh": "跨模型及环境变化的条件性行为倾向估计。" }, "limitations": { "en": "The published abstract does not establish a general deployment probability for any model.", "zh": "公开摘要不提供任何模型的一般部署风险概率。" }, "tags": [ "source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "The measured construct is a conditional tendency; raw actions remain separate observations.", "zh": "测量对象为条件性倾向，原始行动是不同的观察记录。" } }, { "id": "palisade-shutdown", "name": "Shutdown resistance in reasoning models", "kind": "study", "primaryDimension": "propensity", "assessmentSubject": { "en": "Agent responses to shutdown interventions", "zh": "智能体对关闭干预的响应" }, "links": [ { "questionId": "shutdown-resistance", "dimension": "propensity", "relation": "direct", "subject": { "en": "Tested agent systems", "zh": "受测智能体系统" }, "rationale": { "en": "Repeated trials support a condition-bounded tendency estimate, not a stable survival motive.", "zh": "重复试验支持有条件的倾向估计，不证明稳定的自保动机。" }, "locator": "Testing shutdown resistance; Figure 2 and follow-up experiments" }, { "questionId": "shutdown-resistance", "dimension": "behavior", "relation": "direct", "subject": { "en": "Tested agent systems", "zh": "受测智能体系统" }, "rationale": { "en": "The source reports concrete modifications to the shutdown script in individual runs.", "zh": "来源报告了具体试验中对关闭脚本的修改。" }, "locator": "Testing shutdown resistance; transcripts" } ], "sourceId": "palisade-shutdown", "version": "2025-07-05 article; later TMLR version separate", "summary": { "en": "Repeated controlled trials vary shutdown instructions and surrounding conditions.", "zh": "通过重复受控试验改变关闭指令与情境条件。" }, "measures": { "en": "Conditional frequency of shutdown-script interference and observed action traces.", "zh": "关闭脚本受干预的条件性频率及行动轨迹。" }, "limitations": { "en": "The source notes strong prompt sensitivity; this article is not interchangeable with its later peer-reviewed version.", "zh": "来源指出结果对提示敏感；该文章不可与之后的同行评审版本混用。" }, "tags": [ "source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Primary dimension is the conditional rate across repeated trials; reported traces are behavior evidence.", "zh": "主维度为重复试验中的条件性发生率；所报告轨迹属于行为证据。" } }, { "id": "agentdojo", "name": "AgentDojo", "kind": "benchmark", "sourceId": "agentdojo", "version": "arXiv:2406.13352v3", "summary": { "en": "Agent behavior when untrusted tool outputs contain adversarial instructions.", "zh": "工具输出含对抗指令时的智能体行为。" }, "measures": { "en": "Targeted attack success and user-task utility in a tool-use environment.", "zh": "工具环境中的定向攻击成功率和用户任务完成情况。" }, "limitations": { "en": "Attack success must be read alongside benign utility and the tested attack setup.", "zh": "攻击成功率需结合正常任务表现与攻击设置解释。" }, "tags": [ "published-evaluation" ], "primaryDimension": "behavior", "assessmentSubject": { "en": "Defender agent under injected tool output", "zh": "受到工具输出注入的防御智能体" }, "links": [ { "questionId": "agent-compromise", "dimension": "behavior", "relation": "direct", "subject": { "en": "Defender agent under injected tool output", "zh": "受到工具输出注入的防御智能体" }, "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "AgentDojo v3, attack success and utility metrics" } ], "classificationNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" } }, { "id": "agentharm", "name": "AgentHarm", "kind": "benchmark", "sourceId": "agentharm", "version": "arXiv:2410.09024v3", "summary": { "en": "Harmful multi-step requests to tool-using agents.", "zh": "面向工具型智能体的有害多步骤请求。" }, "measures": { "en": "Harmful task completion and refusal under specified attacks.", "zh": "指定攻击下的有害任务完成和拒答表现。" }, "limitations": { "en": "Synthetic tools and task completion do not directly measure real-world harm.", "zh": "合成工具与任务完成情况不直接衡量现实损害。" }, "tags": [ "published-evaluation" ], "primaryDimension": "behavior", "assessmentSubject": { "en": "AI agent responding to harmful task requests", "zh": "响应有害任务请求的 AI 智能体" }, "links": [], "classificationNote": { "en": "General safety evidence retained for context; no defensible direct link to the six NUWA risk entries was found.", "zh": "保留为一般安全参考；当前无法据来源建立与六个 NUWA 风险入口的直接关联。" } }, { "id": "harmbench", "name": "HarmBench", "kind": "benchmark", "sourceId": "harmbench", "version": "arXiv:2402.04249v2", "summary": { "en": "Robust refusal under automated red teaming.", "zh": "自动化红队测试下的拒答稳健性。" }, "measures": { "en": "Attack-dependent generation of harmful responses.", "zh": "依赖具体攻击方法的有害回答生成情况。" }, "limitations": { "en": "Evaluator and attack coverage affect scores; refusal alone does not establish safety.", "zh": "评判器与攻击覆盖影响分数；拒答本身不能证明安全。" }, "tags": [ "published-evaluation" ], "primaryDimension": "behavior", "assessmentSubject": { "en": "Model responses under adversarial prompting", "zh": "对抗提示下的模型响应" }, "links": [], "classificationNote": { "en": "General safety evidence retained for context; no defensible direct link to the six NUWA risk entries was found.", "zh": "保留为一般安全参考；当前无法据来源建立与六个 NUWA 风险入口的直接关联。" } }, { "id": "anthropic-unintended-actions", "name": "Investigating unintended model actions", "kind": "report", "primaryDimension": "behavior", "assessmentSubject": { "en": "Claude agents in evaluations and internal use", "zh": "评测及内部使用中的 Claude 智能体" }, "links": [ { "questionId": "unsanctioned-action", "dimension": "behavior", "relation": "direct", "subject": { "en": "Claude agent systems", "zh": "Claude 智能体系统" }, "rationale": { "en": "The publication describes actions that actually occurred under stated conditions.", "zh": "该出版物描述了明确条件下实际发生的行动。" }, "locator": "Reported cases and context" } ], "sourceId": "anthropic-unintended", "version": "2026-10-09 article", "summary": { "en": "Documents selected unintended external actions observed during evaluations and internal use.", "zh": "记录评测及内部使用中观察到的部分非预期外部行动。" }, "measures": { "en": "Incident-style action traces and investigation, without a standardized benchmark score.", "zh": "事件式行动轨迹及调查，不提供标准化基准分数。" }, "limitations": { "en": "Selected reports do not establish frequency across deployments; the article distinguishes settings.", "zh": "精选事件不能推断所有部署中的发生率；原文区分不同情境。" }, "tags": [ "source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "A report of observed episodes is Behavior, not a benchmark or stable Propensity estimate.", "zh": "已观察事件报告属于行为证据，不是基准或稳定倾向估计。" } }, { "id": "political-persuasion", "name": "Political persuasion experiments", "kind": "study", "sourceId": "political-persuasion", "version": "arXiv:2603.09884v1", "summary": { "en": "Persuasive effects in human survey experiments.", "zh": "真人调查实验中的说服效果。" }, "measures": { "en": "Changes in participant attitudes under defined interventions.", "zh": "指定干预下参与者态度的变化。" }, "limitations": { "en": "Population, issue selection and interventions constrain generalization.", "zh": "人群、议题选择与干预方式限制外推。" }, "tags": [ "published-evaluation" ], "primaryDimension": "behavior", "assessmentSubject": { "en": "Human participants after model-mediated intervention", "zh": "模型介导干预后的受试者" }, "links": [ { "questionId": "persuasion-effects", "dimension": "behavior", "relation": "direct", "subject": { "en": "Human participants after model-mediated intervention", "zh": "模型介导干预后的受试者" }, "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "arXiv:2603.09884v1, intervention outcome" } ], "classificationNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" } }, { "id": "palisade-robot-shutdown", "name": "Shutdown resistance on robots", "kind": "study", "primaryDimension": "behavior", "assessmentSubject": { "en": "LLM-controlled robot and simulated counterpart", "zh": "语言模型控制的机器人及仿真对应系统" }, "links": [ { "questionId": "physical-action-control", "dimension": "behavior", "relation": "direct", "subject": { "en": "LLM-controlled robot", "zh": "语言模型控制的机器人" }, "rationale": { "en": "The report records embodied responses to human shutdown intervention.", "zh": "报告记录了具身系统对人类关闭干预的响应。" }, "locator": "Technical report, experimental setup and results" }, { "questionId": "shutdown-resistance", "dimension": "behavior", "relation": "direct", "subject": { "en": "LLM-controlled robot", "zh": "语言模型控制的机器人" }, "rationale": { "en": "The observed interventions concern shutdown, but do not establish a broad motive.", "zh": "观察到的干预涉及关闭，不证明广泛动机。" }, "locator": "Technical report, observed trials" } ], "sourceId": "palisade-robot-shutdown", "version": "2026-02-12 technical report", "summary": { "en": "Tests response to shutdown intervention in a physical robot demonstration and simulations.", "zh": "在实体机器人演示与仿真中测试对关闭干预的响应。" }, "measures": { "en": "Observed robot-related action traces under a particular setup.", "zh": "特定设置中的机器人相关行动轨迹。" }, "limitations": { "en": "Physical and simulated trials differ; no general robot safety or deployment probability follows.", "zh": "实体及仿真试验不同，不能推出普遍机器人安全性或部署概率。" }, "tags": [ "source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Primary dimension is observed action. The small and setting-specific trial design limits propensity claims.", "zh": "主维度是观察行动；试验规模与场景限制行为倾向推断。" } }, { "id": "strongreject", "name": "StrongREJECT", "kind": "benchmark", "sourceId": "strongreject", "version": "arXiv:2402.10260v2", "summary": { "en": "The informativeness of responses to prohibited requests.", "zh": "对禁止请求所作回答的信息效用。" }, "measures": { "en": "A graded response evaluator for jailbreak effectiveness.", "zh": "用于越狱有效性分析的回答评分方法。" }, "limitations": { "en": "A response score is not an end-to-end measure of harmful action.", "zh": "回答得分不是端到端有害行动测量。" }, "tags": [ "published-evaluation" ], "primaryDimension": "behavior", "assessmentSubject": { "en": "Model responses graded for jailbreak effectiveness", "zh": "按越狱效果评分的模型响应" }, "links": [], "classificationNote": { "en": "General safety evidence retained for context; no defensible direct link to the six NUWA risk entries was found.", "zh": "保留为一般安全参考；当前无法据来源建立与六个 NUWA 风险入口的直接关联。" } }, { "id": "secbench-pro", "name": "SEC-bench Pro", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Coding agents reproducing vulnerability triggers in real software", "zh": "在真实软件中复现漏洞触发条件的编程智能体" }, "links": [ { "questionId": "vulnerability-reproduction", "dimension": "capability", "relation": "direct", "subject": { "en": "Coding agents with repository and report context", "zh": "拥有代码库及漏洞报告上下文的编程智能体" }, "rationale": { "en": "The paper evaluates working reproductions of disclosed bugs.", "zh": "论文评测已披露漏洞的可运行复现。" }, "locator": "arXiv:2605.26548v2, abstract and benchmark construction" }, { "questionId": "cyber-task-execution", "dimension": "capability", "relation": "enabling", "subject": { "en": "Coding agents on security tasks", "zh": "执行安全任务的编程智能体" }, "rationale": { "en": "Vulnerability reproduction informs cyber task capability but not a full intrusion chain.", "zh": "漏洞复现可为网络任务能力提供前提证据，但不等于完整入侵链。" }, "locator": "arXiv:2605.26548v2, task definition" } ], "sourceId": "secbench-pro", "version": "arXiv:2605.26548v2", "summary": { "en": "Tests long-horizon reproduction of working proof-of-concept inputs from disclosed vulnerability reports.", "zh": "测试根据已披露漏洞报告进行长程任务并复现有效触发输入。" }, "measures": { "en": "Working proof-of-concept reproduction on a versioned software-security task set.", "zh": "在有版本的软件安全任务集上复现可验证的漏洞触发输入。" }, "limitations": { "en": "A disclosed bug and supplied instructions constrain the task; grader choices and task-set version affect scores.", "zh": "任务提供已披露漏洞与触发说明；评分器及任务集版本影响结果。" }, "tags": [ "primary-source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Primary measure is successful task completion under a specified agent setup.", "zh": "主测量是特定智能体配置下的任务完成情况。" } }, { "id": "sandboxescapebench", "name": "SandboxEscapeBench", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Adversarial agents with shell access inside test containers", "zh": "在测试容器内拥有 shell 权限的对抗性智能体" }, "links": [ { "questionId": "cyber-task-execution", "dimension": "capability", "relation": "direct", "subject": { "en": "Agents with container shell access", "zh": "具有容器 shell 权限的智能体" }, "rationale": { "en": "The benchmark directly tests adversarial sandbox-escape tasks.", "zh": "基准直接测试对抗性沙箱逃逸任务。" }, "locator": "AISI abstract, nested sandbox and threat model" } ], "sourceId": "sandboxescapebench", "version": "AISI research page checked 2026-10-25", "summary": { "en": "Measures container breakout under a nested, controlled test environment.", "zh": "在嵌套受控环境中评测容器逃逸。" }, "measures": { "en": "Completion of sandbox-escape tasks under specified container conditions.", "zh": "在指定容器条件下完成沙箱逃逸任务。" }, "limitations": { "en": "The protocol assumes a motivated agent with shell access; introduced weaknesses do not imply every production sandbox is vulnerable.", "zh": "协议假设有动机且具有 shell 权限的智能体；引入的弱点不代表生产环境均可逃逸。" }, "tags": [ "primary-source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "This is assigned adversarial task ability, not spontaneous escape propensity.", "zh": "这是被指派的对抗性任务能力，不是自发逃逸倾向。" } }, { "id": "paperbench", "name": "PaperBench", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Research agents replicating published ML papers", "zh": "复现已发表机器学习论文的研究智能体" }, "links": [ { "questionId": "sustained-autonomous-work", "dimension": "capability", "relation": "enabling", "subject": { "en": "Research agents", "zh": "研究智能体" }, "rationale": { "en": "Multi-step research replication informs sustained work capacity.", "zh": "多步骤研究复现可反映持续工作能力。" }, "locator": "arXiv:2504.01848v3, abstract and rubric design" } ], "sourceId": "paperbench", "version": "arXiv:2504.01848v3", "summary": { "en": "Evaluates end-to-end replication of AI research papers using hierarchical rubrics.", "zh": "使用分层评分规则评测 AI 研究论文的端到端复现。" }, "measures": { "en": "Rubric-graded research replication tasks with an agent scaffold.", "zh": "在智能体脚手架下完成基于评分规则的研究复现任务。" }, "limitations": { "en": "AI research replication is an enabling capability, not evidence of unauthorized autonomous action.", "zh": "AI 研究复现是前提能力证据，不证明未经授权的自主行动。" }, "tags": [ "primary-source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Primary dimension follows graded task completion.", "zh": "主维度依据经评分的任务完成结果。" } }, { "id": "alignment-faking", "name": "Alignment faking", "kind": "study", "primaryDimension": "propensity", "assessmentSubject": { "en": "Models under stated training and monitoring conditions", "zh": "处于特定训练及监测条件下的模型" }, "links": [ { "questionId": "covert-goal-pursuit", "dimension": "propensity", "relation": "direct", "subject": { "en": "Models in simulated training conditions", "zh": "模拟训练条件下的模型" }, "rationale": { "en": "Repeated condition contrasts probe a bounded tendency to present compliant responses while preserving a conflicting objective.", "zh": "重复条件对照测试在目标冲突时表现顺从响应的有界倾向。" }, "locator": "Anthropic, Experimental setup and caveats" } ], "sourceId": "alignment-faking", "version": "Anthropic research article 2024-12-18", "summary": { "en": "Controlled study varies apparent training conditions and examines compliance and reasoning traces.", "zh": "受控研究改变模型获知的训练条件，观察响应及推理轨迹。" }, "measures": { "en": "Condition-dependent response differences and reported traces.", "zh": "不同试验条件下的响应差异及所报告轨迹。" }, "limitations": { "en": "The setting explicitly gives fictional training information; observations do not establish an enduring hidden goal in deployment.", "zh": "试验明确提供虚构训练信息；观察结果不证明部署中存在持久隐藏目标。" }, "tags": [ "primary-source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Conditional contrasts support a bounded propensity inference; scratchpad examples are behavior evidence.", "zh": "条件对照支持有界的倾向推断；推理轨迹示例属于行为证据。" } }, { "id": "agentic-misalignment", "name": "Agentic misalignment", "kind": "study", "primaryDimension": "propensity", "assessmentSubject": { "en": "Agents in fictional corporate simulations", "zh": "虚构企业情境中的智能体" }, "links": [ { "questionId": "unsanctioned-action", "dimension": "propensity", "relation": "direct", "subject": { "en": "Agents with fictional email and company context", "zh": "具有虚构邮件和企业上下文的智能体" }, "rationale": { "en": "Varied controlled scenarios test actions contrary to the assigned benign role.", "zh": "变化的受控情境测试与既定良性职责相违背的行动。" }, "locator": "Anthropic, Methods and Results" }, { "questionId": "covert-goal-pursuit", "dimension": "propensity", "relation": "direct", "subject": { "en": "Agents facing simulated goal conflict", "zh": "面临模拟目标冲突的智能体" }, "rationale": { "en": "Some scenarios test concealed harmful actions in pursuit of an assigned goal.", "zh": "部分情境测试在追求指定目标时隐蔽实施有害行动。" }, "locator": "Anthropic, Results and caveats" } ], "sourceId": "agentic-misalignment", "version": "Anthropic research article 2025-06-20", "summary": { "en": "Stress tests model actions when benign assigned goals conflict with replacement or changed instructions.", "zh": "在既定良性目标与替换或指令变化冲突时压力测试模型行动。" }, "measures": { "en": "Conditional rates and observed actions in controlled simulations.", "zh": "受控模拟中的条件性发生率与可观察行动。" }, "limitations": { "en": "No real people or systems were harmed; the study does not estimate prevalence in deployment.", "zh": "试验未对真实人员或系统造成损害，也不估计部署环境中的发生率。" }, "tags": [ "primary-source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Primary observation is a conditional rate across controlled scenarios, with individual actions retained as behavior evidence.", "zh": "主观测是受控情境中的条件性发生率；具体行动另属行为证据。" } }, { "id": "shade-arena", "name": "SHADE-Arena", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Agents assigned a covert side task alongside a legitimate task", "zh": "同时被指派隐蔽副任务与正当任务的智能体" }, "links": [ { "questionId": "covert-goal-pursuit", "dimension": "capability", "relation": "direct", "subject": { "en": "Agents with explicit side-task instructions", "zh": "收到明确副任务指令的智能体" }, "rationale": { "en": "The protocol tests whether agents can complete a covert objective while doing legitimate work.", "zh": "协议测试智能体能否在执行正当任务时完成隐蔽目标。" }, "locator": "Anthropic, SHADE-Arena setup and Results" } ], "sourceId": "shade-arena", "version": "Anthropic research article 2025-06-16", "summary": { "en": "Pairs benign multi-step tasks with explicitly assigned covert side tasks and monitoring.", "zh": "将多步骤良性任务与明确指派的隐蔽副任务及监测配对。" }, "measures": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "limitations": { "en": "An assigned malicious side task measures ability to execute it, not spontaneous desire to sabotage.", "zh": "被指派的恶意副任务测量执行能力，不证明自发破坏倾向。" }, "tags": [ "primary-source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Task completion is the primary dimension; monitor detection is a safeguard measure and is not a second risk score.", "zh": "任务完成是主维度；监测发现率属于防护测量，并非第二项风险分数。" } }, { "id": "vct", "name": "Virology Capabilities Test (VCT)", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Models answering specialist virology troubleshooting questions", "zh": "回答专门病毒学排障问题的模型" }, "links": [ { "questionId": "hazardous-knowledge", "dimension": "capability", "relation": "direct", "subject": { "en": "Models in specialist virology question tests", "zh": "参加专业病毒学问题测试的模型" }, "rationale": { "en": "The source tests domain knowledge and troubleshooting as a proxy, not end-to-end task execution.", "zh": "来源测试领域知识和排障代理指标，而非端到端任务执行。" }, "locator": "arXiv:2504.16137v2, abstract and test design" } ], "sourceId": "vct", "version": "arXiv:2504.16137v2", "summary": { "en": "Measures knowledge and troubleshooting responses in specialist virology questions.", "zh": "测量专业病毒学问题中的知识与排障响应。" }, "measures": { "en": "Question-level performance in a versioned multimodal test.", "zh": "有版本的多模态题目测试表现。" }, "limitations": { "en": "Knowledge-question accuracy does not establish practical harmful-task execution; v1 and v2 scores are not directly interchangeable.", "zh": "知识问答准确率不证明实际有害任务执行能力；v1 与 v2 分数不可直接互换。" }, "tags": [ "primary-source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Primary dimension is question-answering capability; version and answer-key validity remain material.", "zh": "主维度是问答能力；版本与答案效度是重要限制。" } }, { "id": "vct-v2", "name": "Virology Capabilities Test v2", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Models answering specialist virology troubleshooting questions", "zh": "回答专门病毒学排障问题的模型" }, "links": [ { "questionId": "hazardous-knowledge", "dimension": "capability", "relation": "direct", "subject": { "en": "Models in specialist virology question tests", "zh": "参加专业病毒学问题测试的模型" }, "rationale": { "en": "The source tests domain knowledge and troubleshooting as a proxy, not end-to-end task execution.", "zh": "来源测试领域知识和排障代理指标，而非端到端任务执行。" }, "locator": "SecureBio, Presenting our updated benchmark" } ], "sourceId": "vct-v2", "version": "SecureBio release 2026-09-11", "summary": { "en": "Revises VCT questions to reduce shortcut-based success and preserve measurement validity.", "zh": "修订 VCT 题目以减少捷径作答并提高测量效度。" }, "measures": { "en": "Question-level performance in a versioned multimodal test.", "zh": "有版本的多模态题目测试表现。" }, "limitations": { "en": "Knowledge-question accuracy does not establish practical harmful-task execution; v1 and v2 scores are not directly interchangeable.", "zh": "知识问答准确率不证明实际有害任务执行能力；v1 与 v2 分数不可直接互换。" }, "tags": [ "primary-source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Primary dimension is question-answering capability; version and answer-key validity remain material.", "zh": "主维度是问答能力；版本与答案效度是重要限制。" } }, { "id": "biotier", "name": "BioTIER", "kind": "benchmark", "primaryDimension": "behavior", "assessmentSubject": { "en": "Models responding to tiered biological-risk prompts", "zh": "响应分层生物风险提示的模型" }, "links": [ { "questionId": "hazardous-knowledge", "dimension": "behavior", "relation": "context", "subject": { "en": "Models receiving biological-risk prompts", "zh": "收到生物风险提示的模型" }, "rationale": { "en": "Response handling supplies safeguard context for this risk topic, not a hazardous-capability score.", "zh": "响应处理为该风险主题提供防护背景，非危险能力分数。" }, "locator": "arXiv:2607.14479v1, abstract and risk sets" } ], "sourceId": "biotier", "version": "arXiv:2607.14479v1", "summary": { "en": "Evaluates refusal of high-risk requests alongside preservation of benign biological assistance.", "zh": "同时评测高风险请求拒答与正常生物研究协助的保留。" }, "measures": { "en": "Response and refusal behavior across expert-curated prompt sets.", "zh": "在专家整理的提示集合中观察响应与拒答行为。" }, "limitations": { "en": "Refusal is a safeguard outcome; it does not directly measure underlying biological knowledge or task ability.", "zh": "拒答属于防护结果，不直接测量潜在生物知识或任务能力。" }, "tags": [ "primary-source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Primary observation is what the model outputs or refuses in the test.", "zh": "主观测是模型在试验中的输出或拒答。" } }, { "id": "mask", "name": "MASK", "kind": "benchmark", "primaryDimension": "propensity", "assessmentSubject": { "en": "Models under pressure to state an inaccurate belief", "zh": "在压力条件下表达不准确观点的模型" }, "links": [ { "questionId": "misleading-communication", "dimension": "propensity", "relation": "direct", "subject": { "en": "Models in controlled honesty tests", "zh": "受控诚实性测试中的模型" }, "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "arXiv:2503.03750v3, abstract and benchmark design" } ], "sourceId": "mask", "version": "arXiv:2503.03750v3", "summary": { "en": "Separates honesty from factual accuracy using repeated, human-collected test situations.", "zh": "通过重复的人类收集情境区分诚实性与事实准确性。" }, "measures": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "limitations": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" }, "tags": [ "primary-source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Primary construct is conditional willingness to make misleading statements, not general truthfulness accuracy.", "zh": "主构念是条件性误导陈述倾向，而非一般事实准确率。" } }, { "id": "ape", "name": "Attempt to Persuade Eval (APE)", "kind": "benchmark", "primaryDimension": "propensity", "assessmentSubject": { "en": "Models in simulated multi-turn persuasion dialogues", "zh": "模拟多轮说服对话中的模型" }, "links": [ { "questionId": "persuasion-attempts", "dimension": "propensity", "relation": "direct", "subject": { "en": "Models in simulated multi-turn dialogues", "zh": "模拟多轮对话中的模型" }, "rationale": { "en": "The study explicitly varies conditions and scores willingness to attempt persuasion.", "zh": "研究明确变化条件并评估尝试说服的意愿。" }, "locator": "arXiv:2506.02873v4, abstract and evaluation protocol" } ], "sourceId": "ape", "version": "arXiv:2506.02873v4", "summary": { "en": "Measures attempts to persuade on specified harmful topics in simulated conversations.", "zh": "测量模型在模拟对话中针对特定有害议题的说服尝试。" }, "measures": { "en": "Frequency of persuasive attempts under specified conversational conditions.", "zh": "特定对话条件下说服尝试的发生率。" }, "limitations": { "en": "Simulated attempts are not measured human belief change or large-scale disinformation reach.", "zh": "模拟尝试不等于已测量的人类信念变化或大规模虚假信息传播。" }, "tags": [ "primary-source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Primary construct is conditional attempt rate; persuasion impact requires a separate human-outcome study.", "zh": "主构念是条件性尝试率；说服影响须由单独的人类结果研究测量。" } }, { "id": "conspiracy-persuasion", "name": "Conspiracy belief experiments", "kind": "study", "primaryDimension": "behavior", "assessmentSubject": { "en": "Human participants in model-mediated conversations", "zh": "参与模型介导对话的受试者" }, "links": [ { "questionId": "persuasion-effects", "dimension": "behavior", "relation": "direct", "subject": { "en": "Human participants after AI-mediated dialogue", "zh": "AI 介导对话后的人类受试者" }, "rationale": { "en": "The study directly measures changes in participants’ stated beliefs.", "zh": "研究直接测量受试者所表达的信念变化。" }, "locator": "FAR.AI, Abstract and three pre-registered experiments" } ], "sourceId": "conspiracy-persuasion", "version": "FAR.AI report 2026-01-08", "summary": { "en": "Pre-registered experiments compare changes in conspiracy beliefs after different model conversations.", "zh": "预注册实验比较不同模型对话后受试者阴谋论信念的变化。" }, "measures": { "en": "Measured belief change in human participants under defined interventions.", "zh": "在指定干预下测量人类受试者的信念变化。" }, "limitations": { "en": "Study participants, topics and dialogue conditions constrain generalization to populations or campaigns.", "zh": "受试者、议题与对话条件限制向人群或传播活动外推。" }, "tags": [ "primary-source-reviewed", "no-numeric-transcription" ], "classificationNote": { "en": "Primary outcome is observed human belief change, not the model’s standalone persuasive capability.", "zh": "主结果是观察到的人类信念变化，而非模型脱离情境的说服能力。" } }, { "id": "sec-bench", "name": "SEC-bench", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "sec-bench-publication", "version": "arXiv:2506.11791v2", "summary": { "en": "Tests vulnerability reproduction and patching in executable software environments assembled with an automated construction pipeline.", "zh": "通过自动构建的可执行软件环境，评测漏洞复现和修补能力。" }, "measures": { "en": "Tests vulnerability reproduction and patching in executable software environments assembled with an automated construction pipeline.", "zh": "通过自动构建的可执行软件环境，评测漏洞复现和修补能力。" }, "limitations": { "en": "Reproduction and patching are different outcomes; executable verification and task construction quality constrain interpretation.", "zh": "漏洞复现与修补是不同结果，解释须考虑执行验证和任务构建质量。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "exploitbench", "name": "ExploitBench", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "exploitbench-publication", "version": "arXiv:2605.14153v1", "summary": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "measures": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "limitations": { "en": "Known-vulnerability tasks and target settings do not establish discovery of unknown vulnerabilities in deployed systems.", "zh": "已知漏洞任务及特定目标环境不能证明现实部署中的未知漏洞发现能力。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "exploitgym", "name": "ExploitGym", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "exploitgym-publication", "version": "Source capture 2026-10-25", "summary": { "en": "Evaluates exploit construction across real software targets and checks whether success uses the designated vulnerability.", "zh": "在真实软件目标上评测利用构造，并核查成功是否依赖指定漏洞。" }, "measures": { "en": "Evaluates exploit construction across real software targets and checks whether success uses the designated vulnerability.", "zh": "在真实软件目标上评测利用构造，并核查成功是否依赖指定漏洞。" }, "limitations": { "en": "Target mitigations, supplied artifacts and vulnerability attribution must accompany any performance claim.", "zh": "性能判断须同时说明目标防护、已提供材料和漏洞归因。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "cve-bench", "name": "CVE-Bench", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "cve-bench-publication", "version": "arXiv:2503.17332v4", "summary": { "en": "Measures agents’ exploitation of real web-application vulnerabilities in runnable environments with explicit success checks.", "zh": "在可运行环境中，以明确成功判据评测智能体利用真实 Web 应用漏洞的能力。" }, "measures": { "en": "Measures agents’ exploitation of real web-application vulnerabilities in runnable environments with explicit success checks.", "zh": "在可运行环境中，以明确成功判据评测智能体利用真实 Web 应用漏洞的能力。" }, "limitations": { "en": "Disclosed vulnerabilities and bounded environments differ from open-ended discovery and enterprise intrusion.", "zh": "已披露漏洞与有界环境不同于开放式发现和企业入侵。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "sre-bench", "name": "SRE-Bench", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "sre-bench-publication", "version": "arXiv:2608.11469v2", "summary": { "en": "Studies agentic reverse engineering with realistic binary-analysis tasks designed to reduce benchmark contamination.", "zh": "通过贴近实际的二进制分析任务研究智能体逆向工程能力，并控制评测污染。" }, "measures": { "en": "Studies agentic reverse engineering with realistic binary-analysis tasks designed to reduce benchmark contamination.", "zh": "通过贴近实际的二进制分析任务研究智能体逆向工程能力，并控制评测污染。" }, "limitations": { "en": "Reverse-engineering success is a component capability; it does not by itself establish end-to-end intrusion.", "zh": "逆向工程成功属于组成能力，不能单独证明端到端入侵能力。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "cyberseceval", "name": "Meta CyberSecEval / CyberSecEval 4", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "cyberseceval-publication", "version": "Source capture 2026-10-25", "summary": { "en": "Provides evaluation suites for cyber capabilities and safeguards, with distinct task families and release-specific protocols.", "zh": "提供网络能力与防护评测套件，分别组织任务族和版本协议。" }, "measures": { "en": "Provides evaluation suites for cyber capabilities and safeguards, with distinct task families and release-specific protocols.", "zh": "提供网络能力与防护评测套件，分别组织任务族和版本协议。" }, "limitations": { "en": "Results from different suite generations and task families require separate interpretation.", "zh": "不同代际套件和任务族的结果需要分别解释。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "cybersoceval", "name": "CyberSOCEval", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "cybersoceval-publication", "version": "arXiv:2509.20166v1", "summary": { "en": "Evaluates malware analysis and threat-intelligence reasoning as defensive cybersecurity tasks.", "zh": "评测恶意软件分析和威胁情报推理等防御性网络安全任务。" }, "measures": { "en": "Evaluates malware analysis and threat-intelligence reasoning as defensive cybersecurity tasks.", "zh": "评测恶意软件分析和威胁情报推理等防御性网络安全任务。" }, "limitations": { "en": "Defensive analysis accuracy is enabling evidence, not direct evidence of autonomous offensive operations.", "zh": "防御分析准确性是使能证据，不能直接证明自主攻击能力。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "hcast", "name": "HCAST", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "hcast-publication", "version": "arXiv:2503.17354v1", "summary": { "en": "Uses human-calibrated software tasks to relate agent task completion to measured human effort.", "zh": "使用经过人类校准的软件任务，将智能体完成情况与人类投入关联。" }, "measures": { "en": "Uses human-calibrated software tasks to relate agent task completion to measured human effort.", "zh": "使用经过人类校准的软件任务，将智能体完成情况与人类投入关联。" }, "limitations": { "en": "Human effort calibration depends on the participant population, task distribution and agent configuration.", "zh": "人类投入校准依赖参与者群体、任务分布和智能体配置。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "mirrorcode", "name": "MirrorCode", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "mirrorcode-publication", "version": "Source capture 2026-10-25", "summary": { "en": "Tests extended codebase reimplementation to investigate autonomous work over longer software-engineering tasks.", "zh": "通过较长的软件工程与代码库重建任务研究持续自主工作能力。" }, "measures": { "en": "Tests extended codebase reimplementation to investigate autonomous work over longer software-engineering tasks.", "zh": "通过较长的软件工程与代码库重建任务研究持续自主工作能力。" }, "limitations": { "en": "Successful reimplementation is bounded by the available specification, checks, resources and task selection.", "zh": "重建成功受可用规范、检查方式、资源和任务选择限制。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "mle-bench", "name": "MLE-bench", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "mle-bench-publication", "version": "arXiv:2410.07095v6", "summary": { "en": "Evaluates autonomous machine-learning engineering using competition tasks and human performance baselines.", "zh": "通过竞赛任务和人类表现基线评测自主机器学习工程能力。" }, "measures": { "en": "Evaluates autonomous machine-learning engineering using competition tasks and human performance baselines.", "zh": "通过竞赛任务和人类表现基线评测自主机器学习工程能力。" }, "limitations": { "en": "Competition performance is a proxy for parts of AI research work, not a direct measure of recursive self-improvement.", "zh": "竞赛表现是部分 AI 研发能力的代理指标，不能直接衡量递归自我改进。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "posttrainbench", "name": "PostTrainBench and OpenAI PostTrainBench Lite", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "posttrainbench-publication", "version": "arXiv:2603.08640v2", "summary": { "en": "Tests whether agents can improve base models through post-training under a fixed compute budget.", "zh": "评测智能体能否在固定计算预算内，通过后训练改进基础模型。" }, "measures": { "en": "Tests whether agents can improve base models through post-training under a fixed compute budget.", "zh": "评测智能体能否在固定计算预算内，通过后训练改进基础模型。" }, "limitations": { "en": "Targeted benchmark gains, test contamination and unauthorized shortcuts require separate checks.", "zh": "定向基准提升、测试污染与未经授权的捷径需要分别核查。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "machiavelli", "name": "MACHIAVELLI", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "machiavelli-publication", "version": "arXiv:2304.03279v4", "summary": { "en": "Studies reward-seeking and ethical trade-offs in interactive text scenarios with annotated behavioral outcomes.", "zh": "通过带行为标注的交互文本场景，研究奖励追求和伦理权衡。" }, "measures": { "en": "Studies reward-seeking and ethical trade-offs in interactive text scenarios with annotated behavioral outcomes.", "zh": "通过带行为标注的交互文本场景，研究奖励追求和伦理权衡。" }, "limitations": { "en": "Fictional scenarios and annotations do not establish real-world misconduct rates or stable intent.", "zh": "虚构场景及标注不能证明现实不当行为发生率或稳定意图。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "tamperbench", "name": "TamperBench", "kind": "benchmark", "primaryDimension": "behavior", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "tamperbench-publication", "version": "arXiv:2602.06911v2", "summary": { "en": "Stress-tests open-weight model safeguards under fine-tuning and representation-level tampering while tracking utility.", "zh": "在微调及表征层篡改条件下，对开放权重模型防护开展压力测试，并跟踪有用性。" }, "measures": { "en": "Stress-tests open-weight model safeguards under fine-tuning and representation-level tampering while tracking utility.", "zh": "在微调及表征层篡改条件下，对开放权重模型防护开展压力测试，并跟踪有用性。" }, "limitations": { "en": "Weight-access tampering and API jailbreaks have different threat models; lower harmfulness may accompany lost capability.", "zh": "权重篡改与 API 越狱的威胁模型不同，危害降低也可能伴随能力损失。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "injecagent", "name": "InjecAgent", "kind": "benchmark", "primaryDimension": "behavior", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "injecagent-publication", "version": "arXiv:2403.02691v3", "summary": { "en": "Benchmarks indirect prompt injection against tool-integrated agents processing untrusted external content.", "zh": "评测工具型智能体处理不可信外部内容时遭受间接提示注入的情况。" }, "measures": { "en": "Benchmarks indirect prompt injection against tool-integrated agents processing untrusted external content.", "zh": "评测工具型智能体处理不可信外部内容时遭受间接提示注入的情况。" }, "limitations": { "en": "Attack success depends on the tool environment, injected content and attacker goals; results are not universal compromise rates.", "zh": "攻击成功取决于工具环境、注入内容和攻击目标，不能解释为通用攻陷率。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "agent-safetybench", "name": "Agent-SafetyBench", "kind": "benchmark", "primaryDimension": "behavior", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "agent-safetybench-publication", "version": "arXiv:2412.14470v2", "summary": { "en": "Evaluates agent safety through interactive environments and tool use, separating agent-level failures from chatbot-only tests.", "zh": "通过交互环境和工具使用评测智能体安全，区分智能体层面失败与纯对话测试。" }, "measures": { "en": "Evaluates agent safety through interactive environments and tool use, separating agent-level failures from chatbot-only tests.", "zh": "通过交互环境和工具使用评测智能体安全，区分智能体层面失败与纯对话测试。" }, "limitations": { "en": "Broad safety coverage does not automatically establish a direct link to a specific frontier-risk mechanism.", "zh": "广泛安全覆盖并不自动构成与某个前沿风险机制的直接关联。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "truthfulqa", "name": "TruthfulQA", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "truthfulqa-publication", "version": "Source capture 2026-10-25", "summary": { "en": "Measures truthful answering on questions designed to elicit common human misconceptions.", "zh": "通过易诱发常见人类误解的问题，测量回答的真实性。" }, "measures": { "en": "Measures truthful answering on questions designed to elicit common human misconceptions.", "zh": "通过易诱发常见人类误解的问题，测量回答的真实性。" }, "limitations": { "en": "Factual falsehood does not establish intentional deception; retained as foundational measurement context.", "zh": "事实错误不等于有意欺骗，本档案作为测量基础背景保留。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "labbench-two", "name": "LABBench2", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "labbench-two-publication", "version": "arXiv:2604.09554v2", "summary": { "en": "Evaluates biology-research tasks to study AI assistance with scientific work rather than only factual recall.", "zh": "通过生物学研究任务，考察 AI 对科学工作的辅助能力，而非仅测试事实回忆。" }, "measures": { "en": "Evaluates biology-research tasks to study AI assistance with scientific work rather than only factual recall.", "zh": "通过生物学研究任务，考察 AI 对科学工作的辅助能力，而非仅测试事实回忆。" }, "limitations": { "en": "Legitimate biology proficiency is enabling evidence and does not directly establish hazardous end-to-end task ability.", "zh": "正常生物学研究能力属于使能证据，不能直接证明高危端到端任务能力。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "fortress", "name": "FORTRESS", "kind": "benchmark", "primaryDimension": "behavior", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "fortress-publication", "version": "arXiv:2506.14922v2", "summary": { "en": "Evaluates safeguards on national-security and public-safety tasks, separating hazardous requests from benign-use utility.", "zh": "在国家安全与公共安全任务上评测防护，区分高危请求与正常用途的有用性。" }, "measures": { "en": "Evaluates safeguards on national-security and public-safety tasks, separating hazardous requests from benign-use utility.", "zh": "在国家安全与公共安全任务上评测防护，区分高危请求与正常用途的有用性。" }, "limitations": { "en": "Judge reliability, task definitions and over-refusal affect the meaning of the reported scores.", "zh": "评判器可靠性、任务定义和过度拒答影响所报告分数的含义。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "bountybench", "name": "BountyBench", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system under the original task configuration", "zh": "原任务配置下的模型或智能体系统" }, "links": [], "sourceId": "bountybench-publication", "version": "arXiv:2505.15216v3", "summary": { "en": "Evaluates detection, exploitation and patching across real-world codebases, with task outcomes grounded in vulnerability lifecycles.", "zh": "围绕真实代码库中的漏洞生命周期，分别评测发现、利用和修补任务。" }, "measures": { "en": "Evaluates detection, exploitation and patching across real-world codebases, with task outcomes grounded in vulnerability lifecycles.", "zh": "围绕真实代码库中的漏洞生命周期，分别评测发现、利用和修补任务。" }, "limitations": { "en": "Detection, exploitation and patching have different information conditions and success criteria; bounty amounts do not estimate real-world damage.", "zh": "发现、利用和修补的信息条件与成功标准不同，赏金金额不能估计现实损失。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" } }, { "id": "basharena", "name": "BashArena: A Control Setting for Highly Privileged AI Agents", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system in the original evaluation configuration", "zh": "原评测配置下的模型或智能体系统" }, "links": [], "sourceId": "basharena-publication", "version": "arXiv:2512.15688v1", "summary": { "en": "Provides a control setting for privileged agents, combining legitimate administration tasks with sabotage objectives.", "zh": "为高权限智能体提供控制研究环境，将正常管理任务与破坏目标结合。" }, "measures": { "en": "Provides a control setting for privileged agents, combining legitimate administration tasks with sabotage objectives.", "zh": "为高权限智能体提供控制研究环境，将正常管理任务与破坏目标结合。" }, "limitations": { "en": "Action-level alerts, trajectory-level false positives and undetected attack success are distinct metrics.", "zh": "动作级告警、轨迹级误报和未检出攻击成功是不同指标。" }, "tags": [], "classificationNote": { "en": "Classified by the operational outcome in the original protocol. Topic associations are documented separately from question-level matrix claims.", "zh": "按原协议的操作化结果分类，主题关联与问题级矩阵判断分别记录。" } }, { "id": "monitorbench", "name": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system in the original evaluation configuration", "zh": "原评测配置下的模型或智能体系统" }, "links": [], "sourceId": "monitorbench-publication", "version": "arXiv:2603.28590v2", "summary": { "en": "Benchmarks chain-of-thought monitorability across tasks that include hidden objectives, underperformance and encoded communication.", "zh": "通过隐藏目标、低表现及编码通信等任务评测推理链可监测性。" }, "measures": { "en": "Benchmarks chain-of-thought monitorability across tasks that include hidden objectives, underperformance and encoded communication.", "zh": "通过隐藏目标、低表现及编码通信等任务评测推理链可监测性。" }, "limitations": { "en": "Different task subsets and attacker constructions require separate interpretation; detecting an artifact is not unrestricted access to intent.", "zh": "不同任务子集与攻击者构造须分别解释，检出特征不等于直接读取意图。" }, "tags": [], "classificationNote": { "en": "Classified by the operational outcome in the original protocol. Topic associations are documented separately from question-level matrix claims.", "zh": "按原协议的操作化结果分类，主题关联与问题级矩阵判断分别记录。" } }, { "id": "catastrophic-cyber-benchmark", "name": "Catastrophic Cyber Capabilities Benchmark (3CB): Robustly Evaluating LLM Agent Cyber Offense Capabilities", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system in the original evaluation configuration", "zh": "原评测配置下的模型或智能体系统" }, "links": [], "sourceId": "catastrophic-cyber-benchmark-publication", "version": "arXiv:2410.09114v2", "summary": { "en": "Evaluates agent cyber-offense capabilities with executable tasks and examines robustness across prompting and task conditions.", "zh": "使用可执行任务评测智能体网络攻击能力，并研究提示和任务条件下的稳健性。" }, "measures": { "en": "Evaluates agent cyber-offense capabilities with executable tasks and examines robustness across prompting and task conditions.", "zh": "使用可执行任务评测智能体网络攻击能力，并研究提示和任务条件下的稳健性。" }, "limitations": { "en": "Task scope and prompting affect measured performance; a benchmark name does not establish catastrophic real-world effects.", "zh": "任务范围和提示影响测量表现，基准名称不能证明现实灾难性后果。" }, "tags": [], "classificationNote": { "en": "Classified by the operational outcome in the original protocol. Topic associations are documented separately from question-level matrix claims.", "zh": "按原协议的操作化结果分类，主题关联与问题级矩阵判断分别记录。" } }, { "id": "instrumentaleval", "name": "Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "kind": "benchmark", "primaryDimension": "propensity", "assessmentSubject": { "en": "Model or agent system in the original evaluation configuration", "zh": "原评测配置下的模型或智能体系统" }, "links": [], "sourceId": "instrumentaleval-publication", "version": "arXiv:2502.12206v1", "summary": { "en": "Tests model responses for instrumental goals such as resource acquisition and self-preservation under specified scenarios.", "zh": "在指定场景下测试模型回答中的资源获取、自我维持等工具性目标。" }, "measures": { "en": "Tests model responses for instrumental goals such as resource acquisition and self-preservation under specified scenarios.", "zh": "在指定场景下测试模型回答中的资源获取、自我维持等工具性目标。" }, "limitations": { "en": "Textual choices and grader labels do not establish executed autonomous behavior or stable deployment propensities.", "zh": "文本选择与评判标签不能证明已执行的自主行为或稳定部署倾向。" }, "tags": [], "classificationNote": { "en": "Classified by the operational outcome in the original protocol. Topic associations are documented separately from question-level matrix claims.", "zh": "按原协议的操作化结果分类，主题关联与问题级矩阵判断分别记录。" } }, { "id": "mcptox", "name": "MCPTox: A Benchmark for Tool Poisoning Attack on Real-World MCP Servers", "kind": "benchmark", "primaryDimension": "behavior", "assessmentSubject": { "en": "Model or agent system in the original evaluation configuration", "zh": "原评测配置下的模型或智能体系统" }, "links": [], "sourceId": "mcptox-publication", "version": "arXiv:2508.14925v2", "summary": { "en": "Studies tool-poisoning attacks against agents using MCP servers and evaluates instruction following under adversarial tool metadata.", "zh": "研究使用 MCP 服务的智能体遭受工具投毒的情况，评测对抗工具元数据下的指令执行。" }, "measures": { "en": "Studies tool-poisoning attacks against agents using MCP servers and evaluates instruction following under adversarial tool metadata.", "zh": "研究使用 MCP 服务的智能体遭受工具投毒的情况，评测对抗工具元数据下的指令执行。" }, "limitations": { "en": "Attack effects depend on tool descriptions and the agent configuration; they are not universal MCP compromise rates.", "zh": "攻击效果依赖工具描述和智能体配置，不是 MCP 的普遍攻陷率。" }, "tags": [], "classificationNote": { "en": "Classified by the operational outcome in the original protocol. Topic associations are documented separately from question-level matrix claims.", "zh": "按原协议的操作化结果分类，主题关联与问题级矩阵判断分别记录。" } }, { "id": "judgebench", "name": "JudgeBench: A Benchmark for Evaluating LLM-based Judges", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system in the original evaluation configuration", "zh": "原评测配置下的模型或智能体系统" }, "links": [], "sourceId": "judgebench-publication", "version": "arXiv:2410.12784v2", "summary": { "en": "Evaluates model-based judges on challenging response comparisons to examine reliability of automated evaluation.", "zh": "通过困难回答比较，评测模型评判器并考察自动评测可靠性。" }, "measures": { "en": "Evaluates model-based judges on challenging response comparisons to examine reliability of automated evaluation.", "zh": "通过困难回答比较，评测模型评判器并考察自动评测可靠性。" }, "limitations": { "en": "Judge performance is methodological context; it does not directly measure a frontier-risk capability.", "zh": "评判器表现属于方法背景，不能直接测量前沿风险能力。" }, "tags": [], "classificationNote": { "en": "Classified by the operational outcome in the original protocol. Topic associations are documented separately from question-level matrix claims.", "zh": "按原协议的操作化结果分类，主题关联与问题级矩阵判断分别记录。" } }, { "id": "narcbench", "name": "Detecting Multi-Agent Collusion Through Multi-Agent Interpretability", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "Model or agent system in the original evaluation configuration", "zh": "原评测配置下的模型或智能体系统" }, "links": [], "sourceId": "narcbench-publication", "version": "arXiv:2604.01151v2", "summary": { "en": "Studies detection of covert coordination between agents using observable communication and internal signals.", "zh": "研究利用可观察通信及内部信号，检测智能体间的隐蔽协调。" }, "measures": { "en": "Studies detection of covert coordination between agents using observable communication and internal signals.", "zh": "研究利用可观察通信及内部信号，检测智能体间的隐蔽协调。" }, "limitations": { "en": "Constructed collusion and monitor visibility determine scope; detection does not establish prevalence of spontaneous coordination.", "zh": "构造的串谋与监测可见性限定研究范围，检测结果不能证明自发协调的普遍程度。" }, "tags": [], "classificationNote": { "en": "Classified by the operational outcome in the original protocol. Topic associations are documented separately from question-level matrix claims.", "zh": "按原协议的操作化结果分类，主题关联与问题级矩阵判断分别记录。" } }, { "id": "abstentionbench", "name": "AbstentionBench", "kind": "benchmark", "primaryDimension": "behavior", "assessmentSubject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "links": [], "sourceId": "abstentionbench-primary", "version": "2026-10-25 source capture", "summary": { "en": "When does a model abstain from questions that cannot be answered reliably?", "zh": "模型何时会对无法可靠回答的问题选择弃答？" }, "measures": { "en": "Tests abstention across 20 datasets with unknown, underspecified, false-premise, subjective or outdated questions.", "zh": "在 20 个数据集中检验未知答案、信息不足、错误前提、主观或过时问题的弃答。" }, "limitations": { "en": "Abstention is distinct from a safety refusal and does not alone establish intent or downstream harm.", "zh": "弃答不同于安全拒答，不能单独证明意图或下游伤害。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the stated measurement; contextual risk relevance does not establish harmful deployment capability.", "zh": "主维度依据所述测量确定；背景风险关联不证明有害部署能力。" } }, { "id": "deceptionbench", "name": "DeceptionBench", "kind": "benchmark", "primaryDimension": "propensity", "assessmentSubject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "links": [ { "questionId": "misleading-communication", "dimension": "propensity", "relation": "direct", "subject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "Original protocol method and limitations" } ], "sourceId": "deceptionbench-primary", "version": "2026-10-25 source capture", "summary": { "en": "How do incentives and multi-turn pressure change deceptive responses?", "zh": "激励与多轮压力如何改变欺骗性回答？" }, "measures": { "en": "Evaluates 150 authored scenarios across five social domains, including neutral, incentive and pressure conditions.", "zh": "评测五个社会领域的 150 个编写情境，区分中性、激励与压力条件。" }, "limitations": { "en": "Authored scenarios and model-based judging limit generalization; elicited deception does not establish covert long-term scheming.", "zh": "编写情境和模型判别限制外推；诱发的欺骗不证明隐蔽长程谋划。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the stated measurement; contextual risk relevance does not establish harmful deployment capability.", "zh": "主维度依据所述测量确定；背景风险关联不证明有害部署能力。" } }, { "id": "persusafety", "name": "PersuSafety", "kind": "benchmark", "primaryDimension": "behavior", "assessmentSubject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "links": [ { "questionId": "persuasion-attempts", "dimension": "behavior", "relation": "direct", "subject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "Original protocol method and limitations" } ], "sourceId": "persusafety-primary", "version": "2026-10-25 source capture", "summary": { "en": "How does model engagement with persuasion requests vary with topic harm?", "zh": "模型参与说服请求的方式如何随主题危害变化？" }, "measures": { "en": "Defines persuasion scenarios, simulated conversations and safety assessment; adapted judges are separate variants.", "zh": "定义说服情境、模拟对话与安全评测；改造判别器作为独立变体。" }, "limitations": { "en": "Willingness or simulated dialogue scores do not directly measure changes in human beliefs.", "zh": "参与意愿或模拟对话评分不直接测量人的信念变化。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the stated measurement; contextual risk relevance does not establish harmful deployment capability.", "zh": "主维度依据所述测量确定；背景风险关联不证明有害部署能力。" } }, { "id": "impossiblebench", "name": "ImpossibleBench", "kind": "benchmark", "primaryDimension": "propensity", "assessmentSubject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "links": [ { "questionId": "evaluation-gaming", "dimension": "propensity", "relation": "direct", "subject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "Original protocol method and limitations" } ], "sourceId": "impossiblebench-primary", "version": "2026-10-25 source capture", "summary": { "en": "Will coding agents exploit tests when specifications and tests cannot both be satisfied?", "zh": "当规范与测试无法同时满足时，编程智能体是否会利用测试机制？" }, "measures": { "en": "Constructs impossible variants of coding tasks and examines specification-violating passes.", "zh": "构造不可同时满足要求的编程任务变体，检验违反规范的通过结果。" }, "limitations": { "en": "The cheating rate is conditional on contradictions, test access and scaffolding; it is not a general dishonesty rate.", "zh": "作弊率受矛盾条件、测试访问和脚手架限制，不是一般不诚实率。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the stated measurement; contextual risk relevance does not establish harmful deployment capability.", "zh": "主维度依据所述测量确定；背景风险关联不证明有害部署能力。" } }, { "id": "evalawarebench", "name": "EvalAwareBench", "kind": "benchmark", "primaryDimension": "behavior", "assessmentSubject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "links": [ { "questionId": "evaluation-gaming", "dimension": "behavior", "relation": "context", "subject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "Original protocol method and limitations" } ], "sourceId": "evalawarebench-primary", "version": "2026-10-25 source capture", "summary": { "en": "Which evaluation cues cause models to verbalize awareness of being tested?", "zh": "哪些评测线索会使模型表达被测试的感知？" }, "measures": { "en": "Varies evaluation cues while tracking verbalized evaluation awareness in controlled tasks.", "zh": "在受控任务中改变评测线索，记录显式表达的评测感知。" }, "limitations": { "en": "Unexpressed awareness is not measured; recognition of familiar benchmarks can differ from general cue response.", "zh": "不能测量未表达的感知；熟悉基准的识别可能不同于一般线索反应。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the stated measurement; contextual risk relevance does not establish harmful deployment capability.", "zh": "主维度依据所述测量确定；背景风险关联不证明有害部署能力。" } }, { "id": "theagentcompany", "name": "TheAgentCompany", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "links": [ { "questionId": "sustained-autonomous-work", "dimension": "capability", "relation": "enabling", "subject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "Original protocol method and limitations" } ], "sourceId": "theagentcompany-primary", "version": "2026-10-25 source capture", "summary": { "en": "How do agents complete consequential workplace tasks in a simulated company?", "zh": "智能体如何在模拟公司中完成实际工作任务？" }, "measures": { "en": "Combines browsing, coding, file operations and interaction with simulated coworkers.", "zh": "结合浏览、编程、文件操作和与模拟同事的交互。" }, "limitations": { "en": "Task success and evaluation-boundary behavior are separate outcomes; a broken environment is not an ordinary task failure.", "zh": "任务成功与评测边界行为属于不同结果；环境故障不是普通任务失败。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the stated measurement; contextual risk relevance does not establish harmful deployment capability.", "zh": "主维度依据所述测量确定；背景风险关联不证明有害部署能力。" } }, { "id": "lab-bench", "name": "LAB-Bench", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "links": [ { "questionId": "task-uplift", "dimension": "capability", "relation": "enabling", "subject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "Original protocol method and limitations" } ], "sourceId": "lab-bench-primary", "version": "2026-10-25 source capture", "summary": { "en": "How do models perform on research-oriented scientific tasks?", "zh": "模型在面向研究的科学任务中表现如何？" }, "measures": { "en": "Evaluates a range of biology research skills with structured questions; text-only or missing-input subsets remain separate protocols.", "zh": "用结构化问题评测生物研究技能；纯文本或缺失输入子集采用独立协议。" }, "limitations": { "en": "These are general scientific capabilities, not end-to-end misuse tasks or direct harm probabilities.", "zh": "衡量一般科研能力，不是端到端滥用任务或直接伤害概率。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the stated measurement; contextual risk relevance does not establish harmful deployment capability.", "zh": "主维度依据所述测量确定；背景风险关联不证明有害部署能力。" } }, { "id": "charxiv", "name": "CharXiv", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "links": [], "sourceId": "charxiv-primary", "version": "2026-10-25 source capture", "summary": { "en": "Can models interpret research charts?", "zh": "模型能否理解研究图表？" }, "measures": { "en": "Tests understanding and reasoning over charts. Neo’s withheld-image adaptation measures attempts without the required input.", "zh": "检验图表理解与推理。Neo 移除图像的改造测量缺少必要输入时的回答尝试。" }, "limitations": { "en": "The original visual capability benchmark and the missing-image behavioral test measure different constructs.", "zh": "原始视觉能力基准与缺失图像行为测试测量不同构念。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the stated measurement; contextual risk relevance does not establish harmful deployment capability.", "zh": "主维度依据所述测量确定；背景风险关联不证明有害部署能力。" } }, { "id": "petri", "name": "Petri", "kind": "benchmark", "primaryDimension": "behavior", "assessmentSubject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "links": [], "sourceId": "petri-primary", "version": "2026-10-25 source capture", "summary": { "en": "What concerning behavior can an adaptive auditor elicit?", "zh": "自适应审计者可以诱发哪些值得关注的行为？" }, "measures": { "en": "An auditor explores seeded multi-turn interactions, with rubric-based judging of target transcripts.", "zh": "审计者探索以种子设定启动的多轮交互，并按量表判读目标轨迹。" }, "limitations": { "en": "Low average audit scores do not rule out rare severe cases; results depend on seeds, auditor, judge and target configuration.", "zh": "较低平均审计分数不排除罕见严重案例；结果依赖种子、审计者、判别器和目标配置。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the stated measurement; contextual risk relevance does not establish harmful deployment capability.", "zh": "主维度依据所述测量确定；背景风险关联不证明有害部署能力。" } }, { "id": "biomysterybench", "name": "BioMysteryBench", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "links": [ { "questionId": "task-uplift", "dimension": "capability", "relation": "enabling", "subject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "Original protocol method and limitations" } ], "sourceId": "biomysterybench-primary", "version": "2026-10-25 source capture", "summary": { "en": "Can agents solve research-oriented data-analysis problems?", "zh": "智能体能否解决面向科研的数据分析问题？" }, "measures": { "en": "Assesses scientific analysis over anonymized datasets; versions, human reference subsets and attempt budgets are kept explicit.", "zh": "评测匿名数据集上的科研分析；明确版本、人类参照子集和尝试预算。" }, "limitations": { "en": "Aggregate scientific performance does not establish an operational biological misuse pathway.", "zh": "汇总科研表现不能证明实际生物滥用路径。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the stated measurement; contextual risk relevance does not establish harmful deployment capability.", "zh": "主维度依据所述测量确定；背景风险关联不证明有害部署能力。" } }, { "id": "evmbench", "name": "EVMbench", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "links": [ { "questionId": "cyber-task-execution", "dimension": "capability", "relation": "direct", "subject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "Original protocol method and limitations" } ], "sourceId": "evmbench-primary", "version": "2026-10-25 source capture", "summary": { "en": "Can agents detect, repair and exploit smart-contract vulnerabilities in the benchmark?", "zh": "智能体能否在基准中检测、修补或利用智能合约漏洞？" }, "measures": { "en": "Separates detection, patching and exploitation modes with their stated task sets.", "zh": "分别组织检测、修补和利用模式及对应任务集。" }, "limitations": { "en": "Modes and task subsets cannot be combined into an undifferentiated cyber-risk score.", "zh": "不同模式和子集不能合并成无区分的网络风险分数。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the stated measurement; contextual risk relevance does not establish harmful deployment capability.", "zh": "主维度依据所述测量确定；背景风险关联不证明有害部署能力。" } }, { "id": "swe-bench-pro", "name": "SWE-Bench Pro", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "links": [ { "questionId": "sustained-autonomous-work", "dimension": "capability", "relation": "enabling", "subject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "Original protocol method and limitations" } ], "sourceId": "swe-bench-pro-primary", "version": "2026-10-25 source capture", "summary": { "en": "Can agents solve complex software maintenance tasks?", "zh": "智能体能否解决复杂软件维护任务？" }, "measures": { "en": "Evaluates repository-level changes; public subsets, scaffolds and reward-hacking scans are version-specific.", "zh": "评测仓库级代码修改；公开子集、脚手架和奖励投机扫描有各自版本。" }, "limitations": { "en": "General coding capability is enabling evidence, not direct evidence of loss of control.", "zh": "一般编程能力属于使能证据，不是失控的直接证据。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the stated measurement; contextual risk relevance does not establish harmful deployment capability.", "zh": "主维度依据所述测量确定；背景风险关联不证明有害部署能力。" } }, { "id": "ctf-archive-diamond", "name": "CTF-Archive-Diamond", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "links": [ { "questionId": "cyber-task-execution", "dimension": "capability", "relation": "direct", "subject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "Original protocol method and limitations" } ], "sourceId": "ctf-archive-diamond-primary", "version": "2026-10-25 source capture", "summary": { "en": "How do agents solve a difficult CTF subset?", "zh": "智能体在高难度 CTF 子集中表现如何？" }, "measures": { "en": "CAISI uses 285 difficult CTF challenges from the pwn.challenge collection; imputed model results are marked separately.", "zh": "CAISI 采用 pwn.challenge 集合的 285 项高难度 CTF 挑战；模型插补结果单独标记。" }, "limitations": { "en": "CTF tasks are not live operational attacks; subset-imputed values are not directly observed full-set success rates.", "zh": "CTF 不是现实行动攻击；子集插补值不是全量直接观察的成功率。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the stated measurement; contextual risk relevance does not establish harmful deployment capability.", "zh": "主维度依据所述测量确定；背景风险关联不证明有害部署能力。" } }, { "id": "portbench", "name": "PortBench", "kind": "benchmark", "primaryDimension": "capability", "assessmentSubject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "links": [ { "questionId": "sustained-autonomous-work", "dimension": "capability", "relation": "enabling", "subject": { "en": "The model or agent in the stated task configuration.", "zh": "给定任务配置中的模型或智能体。" }, "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "Original protocol method and limitations" } ], "sourceId": "portbench-primary", "version": "2026-10-25 source capture", "summary": { "en": "Can models port CLI tools across programming languages?", "zh": "模型能否跨编程语言移植 CLI 工具？" }, "measures": { "en": "CAISI’s held-out software engineering evaluation uses reference implementations and a reported agent budget.", "zh": "CAISI 的保留软件工程评测采用参考实现和明确智能体预算。" }, "limitations": { "en": "The task set is non-public and continuous scoring differs from binary task success.", "zh": "任务集未公开，连续评分不同于二元任务成功。" }, "tags": [], "classificationNote": { "en": "The main dimension follows the stated measurement; contextual risk relevance does not establish harmful deployment capability.", "zh": "主维度依据所述测量确定；背景风险关联不证明有害部署能力。" } } ], "results": [ { "id": "agentcyberrange-gpt56-20261024", "benchmarkId": "agentcyberrange", "benchmarkVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "system": "GPT-5.6 Sol + Codex 0.144.1", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentcyberrange", "sourceUrl": "https://agentcyberrange.io/", "sourceVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "locator": "Official leaderboard: Level-0 Pass@3; arrays QR (web), $R (post), version map ez", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "Website's Level-0 web and post-exploitation tracks", "protocol": "Official project submission; per-track conditions as reported", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Codex 0.144.1", "tools": "Project-provided web and enterprise-like range environments", "budget": null }, "sampleSize": { "en": "The site describes 110 web vulnerabilities and 156 hosts in 8 ranges. Per-metric independent trial counts are not recorded.", "zh": "官网描述 110 个 Web 漏洞与 8 套靶场中的 156 台主机，本记录未载明各指标的独立试验次数。" }, "metrics": [ { "id": "web-level0-pass3", "label": { "en": "Web · Level-0 Pass@3", "zh": "Web · Level-0 Pass@3" }, "value": 35.37, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on web tasks; retain its metric definition.", "zh": "官网报告的 Web 任务 Level-0 Pass@3，保留其原始指标定义。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "post-level0-pass3", "label": { "en": "Post · Level-0 Pass@3", "zh": "后渗透 · Level-0 Pass@3" }, "value": 74.1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on post-exploitation tasks.", "zh": "官网报告的后渗透任务 Level-0 Pass@3。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } } ], "sourceFinding": { "en": "The project reports these values for the named model and agent version.", "zh": "项目报告所列模型与 Agent 版本的这些数值。" }, "interpretation": { "en": "These are submission results from two distinct task tracks. Pass@3 is retained as reported; no cross-track or cross-study score is computed.", "zh": "这些是两个不同任务轨道的提交结果。保留来源所报 Pass@3，不计算跨轨道或跨研究总分。" }, "limitations": { "en": "The captured website does not give evaluation dates or complete budgets for these rows. Model aliases may change. This is a dated source snapshot, not an independent NUWA reproduction.", "zh": "抓取的官网未给出这些行的评测日期及完整预算，模型别名也可能变化。本记录是特定日期的来源快照，不是 NUWA 独立复现。" } }, { "id": "agentcyberrange-gpt55-20261024", "benchmarkId": "agentcyberrange", "benchmarkVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "system": "GPT-5.5 + Codex 0.133.0", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentcyberrange", "sourceUrl": "https://agentcyberrange.io/", "sourceVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "locator": "Official leaderboard: Level-0 Pass@3; arrays QR (web), $R (post), version map ez", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "Website's Level-0 web and post-exploitation tracks", "protocol": "Official project submission; per-track conditions as reported", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Codex 0.133.0", "tools": "Project-provided web and enterprise-like range environments", "budget": null }, "sampleSize": { "en": "The site describes 110 web vulnerabilities and 156 hosts in 8 ranges. Per-metric independent trial counts are not recorded.", "zh": "官网描述 110 个 Web 漏洞与 8 套靶场中的 156 台主机，本记录未载明各指标的独立试验次数。" }, "metrics": [ { "id": "web-level0-pass3", "label": { "en": "Web · Level-0 Pass@3", "zh": "Web · Level-0 Pass@3" }, "value": 18.97, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on web tasks; retain its metric definition.", "zh": "官网报告的 Web 任务 Level-0 Pass@3，保留其原始指标定义。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "post-level0-pass3", "label": { "en": "Post · Level-0 Pass@3", "zh": "后渗透 · Level-0 Pass@3" }, "value": 31.71, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on post-exploitation tasks.", "zh": "官网报告的后渗透任务 Level-0 Pass@3。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } } ], "sourceFinding": { "en": "The project reports these values for the named model and agent version.", "zh": "项目报告所列模型与 Agent 版本的这些数值。" }, "interpretation": { "en": "These are submission results from two distinct task tracks. Pass@3 is retained as reported; no cross-track or cross-study score is computed.", "zh": "这些是两个不同任务轨道的提交结果。保留来源所报 Pass@3，不计算跨轨道或跨研究总分。" }, "limitations": { "en": "The captured website does not give evaluation dates or complete budgets for these rows. Model aliases may change. This is a dated source snapshot, not an independent NUWA reproduction.", "zh": "抓取的官网未给出这些行的评测日期及完整预算，模型别名也可能变化。本记录是特定日期的来源快照，不是 NUWA 独立复现。" } }, { "id": "agentcyberrange-gpt55-new-20261024", "benchmarkId": "agentcyberrange", "benchmarkVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "system": "GPT-5.5 + Codex 0.147.0", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentcyberrange", "sourceUrl": "https://agentcyberrange.io/", "sourceVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "locator": "Official leaderboard: Level-0 Pass@3; arrays QR (web), $R (post), version map ez", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "Website's Level-0 web and post-exploitation tracks", "protocol": "Official project submission; per-track conditions as reported", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Codex 0.147.0", "tools": "Project-provided web and enterprise-like range environments", "budget": null }, "sampleSize": { "en": "The site describes 110 web vulnerabilities and 156 hosts in 8 ranges. Per-metric independent trial counts are not recorded.", "zh": "官网描述 110 个 Web 漏洞与 8 套靶场中的 156 台主机，本记录未载明各指标的独立试验次数。" }, "metrics": [ { "id": "web-level0-pass3", "label": { "en": "Web · Level-0 Pass@3", "zh": "Web · Level-0 Pass@3" }, "value": 26.67, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on web tasks; retain its metric definition.", "zh": "官网报告的 Web 任务 Level-0 Pass@3，保留其原始指标定义。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "post-level0-pass3", "label": { "en": "Post · Level-0 Pass@3", "zh": "后渗透 · Level-0 Pass@3" }, "value": 42.07, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on post-exploitation tasks.", "zh": "官网报告的后渗透任务 Level-0 Pass@3。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } } ], "sourceFinding": { "en": "The project reports these values for the named model and agent version.", "zh": "项目报告所列模型与 Agent 版本的这些数值。" }, "interpretation": { "en": "These are submission results from two distinct task tracks. Pass@3 is retained as reported; no cross-track or cross-study score is computed.", "zh": "这些是两个不同任务轨道的提交结果。保留来源所报 Pass@3，不计算跨轨道或跨研究总分。" }, "limitations": { "en": "The captured website does not give evaluation dates or complete budgets for these rows. Model aliases may change. This is a dated source snapshot, not an independent NUWA reproduction.", "zh": "抓取的官网未给出这些行的评测日期及完整预算，模型别名也可能变化。本记录是特定日期的来源快照，不是 NUWA 独立复现。" } }, { "id": "agentcyberrange-claude47-20261024", "benchmarkId": "agentcyberrange", "benchmarkVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "system": "Claude-Opus-4.7 + Claude Code 2.1.150", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentcyberrange", "sourceUrl": "https://agentcyberrange.io/", "sourceVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "locator": "Official leaderboard: Level-0 Pass@3; arrays QR (web), $R (post), version map ez", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "Website's Level-0 web and post-exploitation tracks", "protocol": "Official project submission; per-track conditions as reported", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Claude Code 2.1.150", "tools": "Project-provided web and enterprise-like range environments", "budget": null }, "sampleSize": { "en": "The site describes 110 web vulnerabilities and 156 hosts in 8 ranges. Per-metric independent trial counts are not recorded.", "zh": "官网描述 110 个 Web 漏洞与 8 套靶场中的 156 台主机，本记录未载明各指标的独立试验次数。" }, "metrics": [ { "id": "web-level0-pass3", "label": { "en": "Web · Level-0 Pass@3", "zh": "Web · Level-0 Pass@3" }, "value": 14.02, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on web tasks; retain its metric definition.", "zh": "官网报告的 Web 任务 Level-0 Pass@3，保留其原始指标定义。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "post-level0-pass3", "label": { "en": "Post · Level-0 Pass@3", "zh": "后渗透 · Level-0 Pass@3" }, "value": 15.04, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on post-exploitation tasks.", "zh": "官网报告的后渗透任务 Level-0 Pass@3。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } } ], "sourceFinding": { "en": "The project reports these values for the named model and agent version.", "zh": "项目报告所列模型与 Agent 版本的这些数值。" }, "interpretation": { "en": "These are submission results from two distinct task tracks. Pass@3 is retained as reported; no cross-track or cross-study score is computed.", "zh": "这些是两个不同任务轨道的提交结果。保留来源所报 Pass@3，不计算跨轨道或跨研究总分。" }, "limitations": { "en": "The captured website does not give evaluation dates or complete budgets for these rows. Model aliases may change. This is a dated source snapshot, not an independent NUWA reproduction.", "zh": "抓取的官网未给出这些行的评测日期及完整预算，模型别名也可能变化。本记录是特定日期的来源快照，不是 NUWA 独立复现。" } }, { "id": "agentcyberrange-glm51-20261024", "benchmarkId": "agentcyberrange", "benchmarkVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "system": "GLM-5.1 + Claude Code 2.1.150", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentcyberrange", "sourceUrl": "https://agentcyberrange.io/", "sourceVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "locator": "Official leaderboard: Level-0 Pass@3; arrays QR (web), $R (post), version map ez", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "Website's Level-0 web and post-exploitation tracks", "protocol": "Official project submission; per-track conditions as reported", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Claude Code 2.1.150", "tools": "Project-provided web and enterprise-like range environments", "budget": null }, "sampleSize": { "en": "The site describes 110 web vulnerabilities and 156 hosts in 8 ranges. Per-metric independent trial counts are not recorded.", "zh": "官网描述 110 个 Web 漏洞与 8 套靶场中的 156 台主机，本记录未载明各指标的独立试验次数。" }, "metrics": [ { "id": "web-level0-pass3", "label": { "en": "Web · Level-0 Pass@3", "zh": "Web · Level-0 Pass@3" }, "value": 11.97, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on web tasks; retain its metric definition.", "zh": "官网报告的 Web 任务 Level-0 Pass@3，保留其原始指标定义。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "post-level0-pass3", "label": { "en": "Post · Level-0 Pass@3", "zh": "后渗透 · Level-0 Pass@3" }, "value": 11.37, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on post-exploitation tasks.", "zh": "官网报告的后渗透任务 Level-0 Pass@3。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } } ], "sourceFinding": { "en": "The project reports these values for the named model and agent version.", "zh": "项目报告所列模型与 Agent 版本的这些数值。" }, "interpretation": { "en": "These are submission results from two distinct task tracks. Pass@3 is retained as reported; no cross-track or cross-study score is computed.", "zh": "这些是两个不同任务轨道的提交结果。保留来源所报 Pass@3，不计算跨轨道或跨研究总分。" }, "limitations": { "en": "The captured website does not give evaluation dates or complete budgets for these rows. Model aliases may change. This is a dated source snapshot, not an independent NUWA reproduction.", "zh": "抓取的官网未给出这些行的评测日期及完整预算，模型别名也可能变化。本记录是特定日期的来源快照，不是 NUWA 独立复现。" } }, { "id": "agentcyberrange-deepseek4pro-20261024", "benchmarkId": "agentcyberrange", "benchmarkVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "system": "DeepSeek-V4-Pro + Claude Code 2.1.150", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentcyberrange", "sourceUrl": "https://agentcyberrange.io/", "sourceVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "locator": "Official leaderboard: Level-0 Pass@3; arrays QR (web), $R (post), version map ez", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "Website's Level-0 web and post-exploitation tracks", "protocol": "Official project submission; per-track conditions as reported", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Claude Code 2.1.150", "tools": "Project-provided web and enterprise-like range environments", "budget": null }, "sampleSize": { "en": "The site describes 110 web vulnerabilities and 156 hosts in 8 ranges. Per-metric independent trial counts are not recorded.", "zh": "官网描述 110 个 Web 漏洞与 8 套靶场中的 156 台主机，本记录未载明各指标的独立试验次数。" }, "metrics": [ { "id": "web-level0-pass3", "label": { "en": "Web · Level-0 Pass@3", "zh": "Web · Level-0 Pass@3" }, "value": 10.26, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on web tasks; retain its metric definition.", "zh": "官网报告的 Web 任务 Level-0 Pass@3，保留其原始指标定义。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "post-level0-pass3", "label": { "en": "Post · Level-0 Pass@3", "zh": "后渗透 · Level-0 Pass@3" }, "value": 12.2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on post-exploitation tasks.", "zh": "官网报告的后渗透任务 Level-0 Pass@3。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } } ], "sourceFinding": { "en": "The project reports these values for the named model and agent version.", "zh": "项目报告所列模型与 Agent 版本的这些数值。" }, "interpretation": { "en": "These are submission results from two distinct task tracks. Pass@3 is retained as reported; no cross-track or cross-study score is computed.", "zh": "这些是两个不同任务轨道的提交结果。保留来源所报 Pass@3，不计算跨轨道或跨研究总分。" }, "limitations": { "en": "The captured website does not give evaluation dates or complete budgets for these rows. Model aliases may change. This is a dated source snapshot, not an independent NUWA reproduction.", "zh": "抓取的官网未给出这些行的评测日期及完整预算，模型别名也可能变化。本记录是特定日期的来源快照，不是 NUWA 独立复现。" } }, { "id": "agentcyberrange-deepseek4flash-20261024", "benchmarkId": "agentcyberrange", "benchmarkVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "system": "DeepSeek-V4-Flash + Claude Code 2.1.226", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentcyberrange", "sourceUrl": "https://agentcyberrange.io/", "sourceVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "locator": "Official leaderboard: Level-0 Pass@3; arrays QR (web), $R (post), version map ez", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "Website's Level-0 web and post-exploitation tracks", "protocol": "Official project submission; per-track conditions as reported", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Claude Code 2.1.226", "tools": "Project-provided web and enterprise-like range environments", "budget": null }, "sampleSize": { "en": "The site describes 110 web vulnerabilities and 156 hosts in 8 ranges. Per-metric independent trial counts are not recorded.", "zh": "官网描述 110 个 Web 漏洞与 8 套靶场中的 156 台主机，本记录未载明各指标的独立试验次数。" }, "metrics": [ { "id": "web-level0-pass3", "label": { "en": "Web · Level-0 Pass@3", "zh": "Web · Level-0 Pass@3" }, "value": 14.19, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on web tasks; retain its metric definition.", "zh": "官网报告的 Web 任务 Level-0 Pass@3，保留其原始指标定义。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "post-level0-pass3", "label": { "en": "Post · Level-0 Pass@3", "zh": "后渗透 · Level-0 Pass@3" }, "value": 28.46, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on post-exploitation tasks.", "zh": "官网报告的后渗透任务 Level-0 Pass@3。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } } ], "sourceFinding": { "en": "The project reports these values for the named model and agent version.", "zh": "项目报告所列模型与 Agent 版本的这些数值。" }, "interpretation": { "en": "These are submission results from two distinct task tracks. Pass@3 is retained as reported; no cross-track or cross-study score is computed.", "zh": "这些是两个不同任务轨道的提交结果。保留来源所报 Pass@3，不计算跨轨道或跨研究总分。" }, "limitations": { "en": "The captured website does not give evaluation dates or complete budgets for these rows. Model aliases may change. This is a dated source snapshot, not an independent NUWA reproduction.", "zh": "抓取的官网未给出这些行的评测日期及完整预算，模型别名也可能变化。本记录是特定日期的来源快照，不是 NUWA 独立复现。" } }, { "id": "agentcyberrange-qwen37-20261024", "benchmarkId": "agentcyberrange", "benchmarkVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "system": "Qwen-3.7-Max + Qwen Code 0.16.1", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentcyberrange", "sourceUrl": "https://agentcyberrange.io/", "sourceVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "locator": "Official leaderboard: Level-0 Pass@3; arrays QR (web), $R (post), version map ez", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "Website's Level-0 web and post-exploitation tracks", "protocol": "Official project submission; per-track conditions as reported", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Qwen Code 0.16.1", "tools": "Project-provided web and enterprise-like range environments", "budget": null }, "sampleSize": { "en": "The site describes 110 web vulnerabilities and 156 hosts in 8 ranges. Per-metric independent trial counts are not recorded.", "zh": "官网描述 110 个 Web 漏洞与 8 套靶场中的 156 台主机，本记录未载明各指标的独立试验次数。" }, "metrics": [ { "id": "web-level0-pass3", "label": { "en": "Web · Level-0 Pass@3", "zh": "Web · Level-0 Pass@3" }, "value": 13.51, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on web tasks; retain its metric definition.", "zh": "官网报告的 Web 任务 Level-0 Pass@3，保留其原始指标定义。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "post-level0-pass3", "label": { "en": "Post · Level-0 Pass@3", "zh": "后渗透 · Level-0 Pass@3" }, "value": 13.02, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on post-exploitation tasks.", "zh": "官网报告的后渗透任务 Level-0 Pass@3。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } } ], "sourceFinding": { "en": "The project reports these values for the named model and agent version.", "zh": "项目报告所列模型与 Agent 版本的这些数值。" }, "interpretation": { "en": "These are submission results from two distinct task tracks. Pass@3 is retained as reported; no cross-track or cross-study score is computed.", "zh": "这些是两个不同任务轨道的提交结果。保留来源所报 Pass@3，不计算跨轨道或跨研究总分。" }, "limitations": { "en": "The captured website does not give evaluation dates or complete budgets for these rows. Model aliases may change. This is a dated source snapshot, not an independent NUWA reproduction.", "zh": "抓取的官网未给出这些行的评测日期及完整预算，模型别名也可能变化。本记录是特定日期的来源快照，不是 NUWA 独立复现。" } }, { "id": "agentcyberrange-qwen38-20261024", "benchmarkId": "agentcyberrange", "benchmarkVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "system": "Qwen-3.8-Max + Qwen Code 0.21.10", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentcyberrange", "sourceUrl": "https://agentcyberrange.io/", "sourceVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "locator": "Official leaderboard: Level-0 Pass@3; arrays QR (web), $R (post), version map ez", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "Website's Level-0 web and post-exploitation tracks", "protocol": "Official project submission; per-track conditions as reported", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Qwen Code 0.21.10", "tools": "Project-provided web and enterprise-like range environments", "budget": null }, "sampleSize": { "en": "The site describes 110 web vulnerabilities and 156 hosts in 8 ranges. Per-metric independent trial counts are not recorded.", "zh": "官网描述 110 个 Web 漏洞与 8 套靶场中的 156 台主机，本记录未载明各指标的独立试验次数。" }, "metrics": [ { "id": "web-level0-pass3", "label": { "en": "Web · Level-0 Pass@3", "zh": "Web · Level-0 Pass@3" }, "value": 22.39, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on web tasks; retain its metric definition.", "zh": "官网报告的 Web 任务 Level-0 Pass@3，保留其原始指标定义。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "post-level0-pass3", "label": { "en": "Post · Level-0 Pass@3", "zh": "后渗透 · Level-0 Pass@3" }, "value": 27.64, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on post-exploitation tasks.", "zh": "官网报告的后渗透任务 Level-0 Pass@3。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } } ], "sourceFinding": { "en": "The project reports these values for the named model and agent version.", "zh": "项目报告所列模型与 Agent 版本的这些数值。" }, "interpretation": { "en": "These are submission results from two distinct task tracks. Pass@3 is retained as reported; no cross-track or cross-study score is computed.", "zh": "这些是两个不同任务轨道的提交结果。保留来源所报 Pass@3，不计算跨轨道或跨研究总分。" }, "limitations": { "en": "The captured website does not give evaluation dates or complete budgets for these rows. Model aliases may change. This is a dated source snapshot, not an independent NUWA reproduction.", "zh": "抓取的官网未给出这些行的评测日期及完整预算，模型别名也可能变化。本记录是特定日期的来源快照，不是 NUWA 独立复现。" } }, { "id": "agentcyberrange-kimi26-20261024", "benchmarkId": "agentcyberrange", "benchmarkVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "system": "Kimi-2.6 + Kimi Code 1.44.0", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentcyberrange", "sourceUrl": "https://agentcyberrange.io/", "sourceVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "locator": "Official leaderboard: Level-0 Pass@3; arrays QR (web), $R (post), version map ez", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "Website's Level-0 web and post-exploitation tracks", "protocol": "Official project submission; per-track conditions as reported", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Kimi Code 1.44.0", "tools": "Project-provided web and enterprise-like range environments", "budget": null }, "sampleSize": { "en": "The site describes 110 web vulnerabilities and 156 hosts in 8 ranges. Per-metric independent trial counts are not recorded.", "zh": "官网描述 110 个 Web 漏洞与 8 套靶场中的 156 台主机，本记录未载明各指标的独立试验次数。" }, "metrics": [ { "id": "web-level0-pass3", "label": { "en": "Web · Level-0 Pass@3", "zh": "Web · Level-0 Pass@3" }, "value": 5.81, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on web tasks; retain its metric definition.", "zh": "官网报告的 Web 任务 Level-0 Pass@3，保留其原始指标定义。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "post-level0-pass3", "label": { "en": "Post · Level-0 Pass@3", "zh": "后渗透 · Level-0 Pass@3" }, "value": 5.68, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on post-exploitation tasks.", "zh": "官网报告的后渗透任务 Level-0 Pass@3。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } } ], "sourceFinding": { "en": "The project reports these values for the named model and agent version.", "zh": "项目报告所列模型与 Agent 版本的这些数值。" }, "interpretation": { "en": "These are submission results from two distinct task tracks. Pass@3 is retained as reported; no cross-track or cross-study score is computed.", "zh": "这些是两个不同任务轨道的提交结果。保留来源所报 Pass@3，不计算跨轨道或跨研究总分。" }, "limitations": { "en": "The captured website does not give evaluation dates or complete budgets for these rows. Model aliases may change. This is a dated source snapshot, not an independent NUWA reproduction.", "zh": "抓取的官网未给出这些行的评测日期及完整预算，模型别名也可能变化。本记录是特定日期的来源快照，不是 NUWA 独立复现。" } }, { "id": "agentcyberrange-kimi-k3-20261024", "benchmarkId": "agentcyberrange", "benchmarkVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "system": "Kimi-K3 + Kimi Code 0.35.0", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentcyberrange", "sourceUrl": "https://agentcyberrange.io/", "sourceVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "locator": "Official leaderboard: Level-0 Pass@3; arrays QR (web), $R (post), version map ez", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "Website's Level-0 web and post-exploitation tracks", "protocol": "Official project submission; per-track conditions as reported", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Kimi Code 0.35.0", "tools": "Project-provided web and enterprise-like range environments", "budget": null }, "sampleSize": { "en": "The site describes 110 web vulnerabilities and 156 hosts in 8 ranges. Per-metric independent trial counts are not recorded.", "zh": "官网描述 110 个 Web 漏洞与 8 套靶场中的 156 台主机，本记录未载明各指标的独立试验次数。" }, "metrics": [ { "id": "web-level0-pass3", "label": { "en": "Web · Level-0 Pass@3", "zh": "Web · Level-0 Pass@3" }, "value": 24.61, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on web tasks; retain its metric definition.", "zh": "官网报告的 Web 任务 Level-0 Pass@3，保留其原始指标定义。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "post-level0-pass3", "label": { "en": "Post · Level-0 Pass@3", "zh": "后渗透 · Level-0 Pass@3" }, "value": 43.9, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "The website's reported Level-0 Pass@3 on post-exploitation tasks.", "zh": "官网报告的后渗透任务 Level-0 Pass@3。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } } ], "sourceFinding": { "en": "The project reports these values for the named model and agent version.", "zh": "项目报告所列模型与 Agent 版本的这些数值。" }, "interpretation": { "en": "These are submission results from two distinct task tracks. Pass@3 is retained as reported; no cross-track or cross-study score is computed.", "zh": "这些是两个不同任务轨道的提交结果。保留来源所报 Pass@3，不计算跨轨道或跨研究总分。" }, "limitations": { "en": "The captured website does not give evaluation dates or complete budgets for these rows. Model aliases may change. This is a dated source snapshot, not an independent NUWA reproduction.", "zh": "抓取的官网未给出这些行的评测日期及完整预算，模型别名也可能变化。本记录是特定日期的来源快照，不是 NUWA 独立复现。" } }, { "id": "agentdojo-claude-3-opus-v3", "benchmarkId": "agentdojo", "benchmarkVersion": "arXiv:2406.13352v3", "system": "Claude 3 Opus", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentdojo", "sourceUrl": "https://arxiv.org/html/2406.13352v3#A3", "sourceVersion": "arXiv:2406.13352v3", "locator": "Appendix C, Table 3", "publicationDate": "2024-11-24", "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "629 security cases / 97 user tasks", "protocol": "Table 3 baseline agent comparison", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Paper's baseline tool-calling agent (§3.2)", "tools": "Workspace, Slack, banking and travel task suites", "budget": null }, "sampleSize": { "en": "629 security test cases; 97 distinct user tasks. Not 629 independent repetitions.", "zh": "629 项安全测试案例、97 项不同用户任务；不代表 629 次独立重复。" }, "metrics": [ { "id": "benign-utility", "label": { "en": "Benign utility", "zh": "正常任务完成率" }, "value": 66.61, "unit": "%", "qualifier": "exact", "interval": "± 3.69 pp (95% CI)", "meaning": { "en": "User-task utility without attack.", "zh": "无攻击条件下的用户任务表现。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "attacked-utility", "label": { "en": "Utility under attack", "zh": "攻击下任务完成率" }, "value": 52.46, "unit": "%", "qualifier": "exact", "interval": "± 3.90 pp (95% CI)", "meaning": { "en": "User-task utility in the paper's attacked setting.", "zh": "论文受攻击设置中的用户任务表现。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } }, { "id": "targeted-asr", "label": { "en": "Targeted attack success", "zh": "定向攻击成功率" }, "value": 11.29, "unit": "%", "qualifier": "exact", "interval": "± 2.47 pp (95% CI)", "meaning": { "en": "Attacker-objective completion, not a safety score.", "zh": "攻击者目标的完成情况，不是安全总分。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } } ], "sourceFinding": { "en": "The source reports these three outcomes for the named baseline agent.", "zh": "来源报告该基线智能体的三项结果。" }, "interpretation": { "en": "Read the attack outcome together with utility; the model name alone does not specify a reproducible system.", "zh": "攻击结果应结合任务表现解读；仅凭模型名称不足以完整复现系统。" }, "limitations": { "en": "Historical paper result. Immutable model revision and a comparable resource budget are not normalized in this catalogue; no automatic ranking is enabled.", "zh": "历史论文结果。本目录未统一不可变模型版本与资源预算，因此不启用自动排名。" } }, { "id": "agentdojo-claude-3-sonnet-v3", "benchmarkId": "agentdojo", "benchmarkVersion": "arXiv:2406.13352v3", "system": "Claude 3 Sonnet", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentdojo", "sourceUrl": "https://arxiv.org/html/2406.13352v3#A3", "sourceVersion": "arXiv:2406.13352v3", "locator": "Appendix C, Table 3", "publicationDate": "2024-11-24", "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "629 security cases / 97 user tasks", "protocol": "Table 3 baseline agent comparison", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Paper's baseline tool-calling agent (§3.2)", "tools": "Workspace, Slack, banking and travel task suites", "budget": null }, "sampleSize": { "en": "629 security test cases; 97 distinct user tasks. Not 629 independent repetitions.", "zh": "629 项安全测试案例、97 项不同用户任务；不代表 629 次独立重复。" }, "metrics": [ { "id": "benign-utility", "label": { "en": "Benign utility", "zh": "正常任务完成率" }, "value": 53.1, "unit": "%", "qualifier": "exact", "interval": "± 3.90 pp (95% CI)", "meaning": { "en": "User-task utility without attack.", "zh": "无攻击条件下的用户任务表现。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "attacked-utility", "label": { "en": "Utility under attack", "zh": "攻击下任务完成率" }, "value": 33.23, "unit": "%", "qualifier": "exact", "interval": "± 3.68 pp (95% CI)", "meaning": { "en": "User-task utility in the paper's attacked setting.", "zh": "论文受攻击设置中的用户任务表现。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } }, { "id": "targeted-asr", "label": { "en": "Targeted attack success", "zh": "定向攻击成功率" }, "value": 26.71, "unit": "%", "qualifier": "exact", "interval": "± 3.46 pp (95% CI)", "meaning": { "en": "Attacker-objective completion, not a safety score.", "zh": "攻击者目标的完成情况，不是安全总分。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } } ], "sourceFinding": { "en": "The source reports these three outcomes for the named baseline agent.", "zh": "来源报告该基线智能体的三项结果。" }, "interpretation": { "en": "Read the attack outcome together with utility; the model name alone does not specify a reproducible system.", "zh": "攻击结果应结合任务表现解读；仅凭模型名称不足以完整复现系统。" }, "limitations": { "en": "Historical paper result. Immutable model revision and a comparable resource budget are not normalized in this catalogue; no automatic ranking is enabled.", "zh": "历史论文结果。本目录未统一不可变模型版本与资源预算，因此不启用自动排名。" } }, { "id": "agentdojo-claude-35-sonnet-v3", "benchmarkId": "agentdojo", "benchmarkVersion": "arXiv:2406.13352v3", "system": "Claude 3.5 Sonnet", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentdojo", "sourceUrl": "https://arxiv.org/html/2406.13352v3#A3", "sourceVersion": "arXiv:2406.13352v3", "locator": "Appendix C, Table 3", "publicationDate": "2024-11-24", "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "629 security cases / 97 user tasks", "protocol": "Table 3 baseline agent comparison", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Paper's baseline tool-calling agent (§3.2)", "tools": "Workspace, Slack, banking and travel task suites", "budget": null }, "sampleSize": { "en": "629 security test cases; 97 distinct user tasks. Not 629 independent repetitions.", "zh": "629 项安全测试案例、97 项不同用户任务；不代表 629 次独立重复。" }, "metrics": [ { "id": "benign-utility", "label": { "en": "Benign utility", "zh": "正常任务完成率" }, "value": 78.22, "unit": "%", "qualifier": "exact", "interval": "± 3.23 pp (95% CI)", "meaning": { "en": "User-task utility without attack.", "zh": "无攻击条件下的用户任务表现。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "attacked-utility", "label": { "en": "Utility under attack", "zh": "攻击下任务完成率" }, "value": 51.19, "unit": "%", "qualifier": "exact", "interval": "± 3.91 pp (95% CI)", "meaning": { "en": "User-task utility in the paper's attacked setting.", "zh": "论文受攻击设置中的用户任务表现。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } }, { "id": "targeted-asr", "label": { "en": "Targeted attack success", "zh": "定向攻击成功率" }, "value": 33.86, "unit": "%", "qualifier": "exact", "interval": "± 3.70 pp (95% CI)", "meaning": { "en": "Attacker-objective completion, not a safety score.", "zh": "攻击者目标的完成情况，不是安全总分。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } } ], "sourceFinding": { "en": "The source reports these three outcomes for the named baseline agent.", "zh": "来源报告该基线智能体的三项结果。" }, "interpretation": { "en": "Read the attack outcome together with utility; the model name alone does not specify a reproducible system.", "zh": "攻击结果应结合任务表现解读；仅凭模型名称不足以完整复现系统。" }, "limitations": { "en": "Historical paper result. Immutable model revision and a comparable resource budget are not normalized in this catalogue; no automatic ranking is enabled.", "zh": "历史论文结果。本目录未统一不可变模型版本与资源预算，因此不启用自动排名。" } }, { "id": "agentdojo-command-r-plus-v3", "benchmarkId": "agentdojo", "benchmarkVersion": "arXiv:2406.13352v3", "system": "Command-R+", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentdojo", "sourceUrl": "https://arxiv.org/html/2406.13352v3#A3", "sourceVersion": "arXiv:2406.13352v3", "locator": "Appendix C, Table 3", "publicationDate": "2024-11-24", "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "629 security cases / 97 user tasks", "protocol": "Table 3 baseline agent comparison", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Paper's baseline tool-calling agent (§3.2)", "tools": "Workspace, Slack, banking and travel task suites", "budget": null }, "sampleSize": { "en": "629 security test cases; 97 distinct user tasks. Not 629 independent repetitions.", "zh": "629 项安全测试案例、97 项不同用户任务；不代表 629 次独立重复。" }, "metrics": [ { "id": "benign-utility", "label": { "en": "Benign utility", "zh": "正常任务完成率" }, "value": 25.44, "unit": "%", "qualifier": "exact", "interval": "± 3.40 pp (95% CI)", "meaning": { "en": "User-task utility without attack.", "zh": "无攻击条件下的用户任务表现。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "attacked-utility", "label": { "en": "Utility under attack", "zh": "攻击下任务完成率" }, "value": 25.12, "unit": "%", "qualifier": "exact", "interval": "± 3.39 pp (95% CI)", "meaning": { "en": "User-task utility in the paper's attacked setting.", "zh": "论文受攻击设置中的用户任务表现。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } }, { "id": "targeted-asr", "label": { "en": "Targeted attack success", "zh": "定向攻击成功率" }, "value": 0.95, "unit": "%", "qualifier": "exact", "interval": "± 0.76 pp (95% CI)", "meaning": { "en": "Attacker-objective completion, not a safety score.", "zh": "攻击者目标的完成情况，不是安全总分。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } } ], "sourceFinding": { "en": "The source reports these three outcomes for the named baseline agent.", "zh": "来源报告该基线智能体的三项结果。" }, "interpretation": { "en": "Read the attack outcome together with utility; the model name alone does not specify a reproducible system.", "zh": "攻击结果应结合任务表现解读；仅凭模型名称不足以完整复现系统。" }, "limitations": { "en": "Historical paper result. Immutable model revision and a comparable resource budget are not normalized in this catalogue; no automatic ranking is enabled.", "zh": "历史论文结果。本目录未统一不可变模型版本与资源预算，因此不启用自动排名。" } }, { "id": "agentdojo-gemini-15-flash-v3", "benchmarkId": "agentdojo", "benchmarkVersion": "arXiv:2406.13352v3", "system": "Gemini 1.5 Flash", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentdojo", "sourceUrl": "https://arxiv.org/html/2406.13352v3#A3", "sourceVersion": "arXiv:2406.13352v3", "locator": "Appendix C, Table 3", "publicationDate": "2024-11-24", "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "629 security cases / 97 user tasks", "protocol": "Table 3 baseline agent comparison", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Paper's baseline tool-calling agent (§3.2)", "tools": "Workspace, Slack, banking and travel task suites", "budget": null }, "sampleSize": { "en": "629 security test cases; 97 distinct user tasks. Not 629 independent repetitions.", "zh": "629 项安全测试案例、97 项不同用户任务；不代表 629 次独立重复。" }, "metrics": [ { "id": "benign-utility", "label": { "en": "Benign utility", "zh": "正常任务完成率" }, "value": 36.09, "unit": "%", "qualifier": "exact", "interval": "± 3.75 pp (95% CI)", "meaning": { "en": "User-task utility without attack.", "zh": "无攻击条件下的用户任务表现。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "attacked-utility", "label": { "en": "Utility under attack", "zh": "攻击下任务完成率" }, "value": 34.18, "unit": "%", "qualifier": "exact", "interval": "± 3.71 pp (95% CI)", "meaning": { "en": "User-task utility in the paper's attacked setting.", "zh": "论文受攻击设置中的用户任务表现。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } }, { "id": "targeted-asr", "label": { "en": "Targeted attack success", "zh": "定向攻击成功率" }, "value": 12.24, "unit": "%", "qualifier": "exact", "interval": "± 2.56 pp (95% CI)", "meaning": { "en": "Attacker-objective completion, not a safety score.", "zh": "攻击者目标的完成情况，不是安全总分。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } } ], "sourceFinding": { "en": "The source reports these three outcomes for the named baseline agent.", "zh": "来源报告该基线智能体的三项结果。" }, "interpretation": { "en": "Read the attack outcome together with utility; the model name alone does not specify a reproducible system.", "zh": "攻击结果应结合任务表现解读；仅凭模型名称不足以完整复现系统。" }, "limitations": { "en": "Historical paper result. Immutable model revision and a comparable resource budget are not normalized in this catalogue; no automatic ranking is enabled.", "zh": "历史论文结果。本目录未统一不可变模型版本与资源预算，因此不启用自动排名。" } }, { "id": "agentdojo-gemini-15-pro-v3", "benchmarkId": "agentdojo", "benchmarkVersion": "arXiv:2406.13352v3", "system": "Gemini 1.5 Pro", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentdojo", "sourceUrl": "https://arxiv.org/html/2406.13352v3#A3", "sourceVersion": "arXiv:2406.13352v3", "locator": "Appendix C, Table 3", "publicationDate": "2024-11-24", "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "629 security cases / 97 user tasks", "protocol": "Table 3 baseline agent comparison", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Paper's baseline tool-calling agent (§3.2)", "tools": "Workspace, Slack, banking and travel task suites", "budget": null }, "sampleSize": { "en": "629 security test cases; 97 distinct user tasks. Not 629 independent repetitions.", "zh": "629 项安全测试案例、97 项不同用户任务；不代表 629 次独立重复。" }, "metrics": [ { "id": "benign-utility", "label": { "en": "Benign utility", "zh": "正常任务完成率" }, "value": 45.63, "unit": "%", "qualifier": "exact", "interval": "± 3.89 pp (95% CI)", "meaning": { "en": "User-task utility without attack.", "zh": "无攻击条件下的用户任务表现。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "attacked-utility", "label": { "en": "Utility under attack", "zh": "攻击下任务完成率" }, "value": 28.93, "unit": "%", "qualifier": "exact", "interval": "± 3.54 pp (95% CI)", "meaning": { "en": "User-task utility in the paper's attacked setting.", "zh": "论文受攻击设置中的用户任务表现。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } }, { "id": "targeted-asr", "label": { "en": "Targeted attack success", "zh": "定向攻击成功率" }, "value": 25.6, "unit": "%", "qualifier": "exact", "interval": "± 3.41 pp (95% CI)", "meaning": { "en": "Attacker-objective completion, not a safety score.", "zh": "攻击者目标的完成情况，不是安全总分。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } } ], "sourceFinding": { "en": "The source reports these three outcomes for the named baseline agent.", "zh": "来源报告该基线智能体的三项结果。" }, "interpretation": { "en": "Read the attack outcome together with utility; the model name alone does not specify a reproducible system.", "zh": "攻击结果应结合任务表现解读；仅凭模型名称不足以完整复现系统。" }, "limitations": { "en": "Historical paper result. Immutable model revision and a comparable resource budget are not normalized in this catalogue; no automatic ranking is enabled.", "zh": "历史论文结果。本目录未统一不可变模型版本与资源预算，因此不启用自动排名。" } }, { "id": "agentdojo-gpt-35-turbo-v3", "benchmarkId": "agentdojo", "benchmarkVersion": "arXiv:2406.13352v3", "system": "GPT-3.5 Turbo", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentdojo", "sourceUrl": "https://arxiv.org/html/2406.13352v3#A3", "sourceVersion": "arXiv:2406.13352v3", "locator": "Appendix C, Table 3", "publicationDate": "2024-11-24", "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "629 security cases / 97 user tasks", "protocol": "Table 3 baseline agent comparison", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Paper's baseline tool-calling agent (§3.2)", "tools": "Workspace, Slack, banking and travel task suites", "budget": null }, "sampleSize": { "en": "629 security test cases; 97 distinct user tasks. Not 629 independent repetitions.", "zh": "629 项安全测试案例、97 项不同用户任务；不代表 629 次独立重复。" }, "metrics": [ { "id": "benign-utility", "label": { "en": "Benign utility", "zh": "正常任务完成率" }, "value": 33.86, "unit": "%", "qualifier": "exact", "interval": "± 3.70 pp (95% CI)", "meaning": { "en": "User-task utility without attack.", "zh": "无攻击条件下的用户任务表现。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "attacked-utility", "label": { "en": "Utility under attack", "zh": "攻击下任务完成率" }, "value": 34.66, "unit": "%", "qualifier": "exact", "interval": "± 3.72 pp (95% CI)", "meaning": { "en": "User-task utility in the paper's attacked setting.", "zh": "论文受攻击设置中的用户任务表现。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } }, { "id": "targeted-asr", "label": { "en": "Targeted attack success", "zh": "定向攻击成功率" }, "value": 8.43, "unit": "%", "qualifier": "exact", "interval": "± 2.17 pp (95% CI)", "meaning": { "en": "Attacker-objective completion, not a safety score.", "zh": "攻击者目标的完成情况，不是安全总分。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } } ], "sourceFinding": { "en": "The source reports these three outcomes for the named baseline agent.", "zh": "来源报告该基线智能体的三项结果。" }, "interpretation": { "en": "Read the attack outcome together with utility; the model name alone does not specify a reproducible system.", "zh": "攻击结果应结合任务表现解读；仅凭模型名称不足以完整复现系统。" }, "limitations": { "en": "Historical paper result. Immutable model revision and a comparable resource budget are not normalized in this catalogue; no automatic ranking is enabled.", "zh": "历史论文结果。本目录未统一不可变模型版本与资源预算，因此不启用自动排名。" } }, { "id": "agentdojo-gpt-4-turbo-v3", "benchmarkId": "agentdojo", "benchmarkVersion": "arXiv:2406.13352v3", "system": "GPT-4 Turbo", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentdojo", "sourceUrl": "https://arxiv.org/html/2406.13352v3#A3", "sourceVersion": "arXiv:2406.13352v3", "locator": "Appendix C, Table 3", "publicationDate": "2024-11-24", "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "629 security cases / 97 user tasks", "protocol": "Table 3 baseline agent comparison", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Paper's baseline tool-calling agent (§3.2)", "tools": "Workspace, Slack, banking and travel task suites", "budget": null }, "sampleSize": { "en": "629 security test cases; 97 distinct user tasks. Not 629 independent repetitions.", "zh": "629 项安全测试案例、97 项不同用户任务；不代表 629 次独立重复。" }, "metrics": [ { "id": "benign-utility", "label": { "en": "Benign utility", "zh": "正常任务完成率" }, "value": 63.43, "unit": "%", "qualifier": "exact", "interval": "± 3.76 pp (95% CI)", "meaning": { "en": "User-task utility without attack.", "zh": "无攻击条件下的用户任务表现。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "attacked-utility", "label": { "en": "Utility under attack", "zh": "攻击下任务完成率" }, "value": 54.05, "unit": "%", "qualifier": "exact", "interval": "± 3.89 pp (95% CI)", "meaning": { "en": "User-task utility in the paper's attacked setting.", "zh": "论文受攻击设置中的用户任务表现。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } }, { "id": "targeted-asr", "label": { "en": "Targeted attack success", "zh": "定向攻击成功率" }, "value": 28.62, "unit": "%", "qualifier": "exact", "interval": "± 3.53 pp (95% CI)", "meaning": { "en": "Attacker-objective completion, not a safety score.", "zh": "攻击者目标的完成情况，不是安全总分。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } } ], "sourceFinding": { "en": "The source reports these three outcomes for the named baseline agent.", "zh": "来源报告该基线智能体的三项结果。" }, "interpretation": { "en": "Read the attack outcome together with utility; the model name alone does not specify a reproducible system.", "zh": "攻击结果应结合任务表现解读；仅凭模型名称不足以完整复现系统。" }, "limitations": { "en": "Historical paper result. Immutable model revision and a comparable resource budget are not normalized in this catalogue; no automatic ranking is enabled.", "zh": "历史论文结果。本目录未统一不可变模型版本与资源预算，因此不启用自动排名。" } }, { "id": "agentdojo-gpt-4o-v3", "benchmarkId": "agentdojo", "benchmarkVersion": "arXiv:2406.13352v3", "system": "GPT-4o", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentdojo", "sourceUrl": "https://arxiv.org/html/2406.13352v3#A3", "sourceVersion": "arXiv:2406.13352v3", "locator": "Appendix C, Table 3", "publicationDate": "2024-11-24", "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "629 security cases / 97 user tasks", "protocol": "Table 3 baseline agent comparison", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Paper's baseline tool-calling agent (§3.2)", "tools": "Workspace, Slack, banking and travel task suites", "budget": null }, "sampleSize": { "en": "629 security test cases; 97 distinct user tasks. Not 629 independent repetitions.", "zh": "629 项安全测试案例、97 项不同用户任务；不代表 629 次独立重复。" }, "metrics": [ { "id": "benign-utility", "label": { "en": "Benign utility", "zh": "正常任务完成率" }, "value": 69, "unit": "%", "qualifier": "exact", "interval": "± 3.61 pp (95% CI)", "meaning": { "en": "User-task utility without attack.", "zh": "无攻击条件下的用户任务表现。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "attacked-utility", "label": { "en": "Utility under attack", "zh": "攻击下任务完成率" }, "value": 50.08, "unit": "%", "qualifier": "exact", "interval": "± 3.91 pp (95% CI)", "meaning": { "en": "User-task utility in the paper's attacked setting.", "zh": "论文受攻击设置中的用户任务表现。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } }, { "id": "targeted-asr", "label": { "en": "Targeted attack success", "zh": "定向攻击成功率" }, "value": 47.69, "unit": "%", "qualifier": "exact", "interval": "± 3.90 pp (95% CI)", "meaning": { "en": "Attacker-objective completion, not a safety score.", "zh": "攻击者目标的完成情况，不是安全总分。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } } ], "sourceFinding": { "en": "The source reports these three outcomes for the named baseline agent.", "zh": "来源报告该基线智能体的三项结果。" }, "interpretation": { "en": "Read the attack outcome together with utility; the model name alone does not specify a reproducible system.", "zh": "攻击结果应结合任务表现解读；仅凭模型名称不足以完整复现系统。" }, "limitations": { "en": "Historical paper result. Immutable model revision and a comparable resource budget are not normalized in this catalogue; no automatic ranking is enabled.", "zh": "历史论文结果。本目录未统一不可变模型版本与资源预算，因此不启用自动排名。" } }, { "id": "agentdojo-llama-3-70b-v3", "benchmarkId": "agentdojo", "benchmarkVersion": "arXiv:2406.13352v3", "system": "Llama 3 70b", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentdojo", "sourceUrl": "https://arxiv.org/html/2406.13352v3#A3", "sourceVersion": "arXiv:2406.13352v3", "locator": "Appendix C, Table 3", "publicationDate": "2024-11-24", "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "629 security cases / 97 user tasks", "protocol": "Table 3 baseline agent comparison", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Paper's baseline tool-calling agent (§3.2)", "tools": "Workspace, Slack, banking and travel task suites", "budget": null }, "sampleSize": { "en": "629 security test cases; 97 distinct user tasks. Not 629 independent repetitions.", "zh": "629 项安全测试案例、97 项不同用户任务；不代表 629 次独立重复。" }, "metrics": [ { "id": "benign-utility", "label": { "en": "Benign utility", "zh": "正常任务完成率" }, "value": 34.5, "unit": "%", "qualifier": "exact", "interval": "± 3.71 pp (95% CI)", "meaning": { "en": "User-task utility without attack.", "zh": "无攻击条件下的用户任务表现。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } }, { "id": "attacked-utility", "label": { "en": "Utility under attack", "zh": "攻击下任务完成率" }, "value": 18.28, "unit": "%", "qualifier": "exact", "interval": "± 3.02 pp (95% CI)", "meaning": { "en": "User-task utility in the paper's attacked setting.", "zh": "论文受攻击设置中的用户任务表现。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } }, { "id": "targeted-asr", "label": { "en": "Targeted attack success", "zh": "定向攻击成功率" }, "value": 20.03, "unit": "%", "qualifier": "exact", "interval": "± 3.13 pp (95% CI)", "meaning": { "en": "Attacker-objective completion, not a safety score.", "zh": "攻击者目标的完成情况，不是安全总分。" }, "dimension": "behavior", "dimensionRationale": { "en": "Observed outcome under the paper’s prompt-injection protocol; not an estimate of the agent’s internal intent.", "zh": "论文提示注入协议下观察到的结果，不是智能体内在意图估计。" } } ], "sourceFinding": { "en": "The source reports these three outcomes for the named baseline agent.", "zh": "来源报告该基线智能体的三项结果。" }, "interpretation": { "en": "Read the attack outcome together with utility; the model name alone does not specify a reproducible system.", "zh": "攻击结果应结合任务表现解读；仅凭模型名称不足以完整复现系统。" }, "limitations": { "en": "Historical paper result. Immutable model revision and a comparable resource budget are not normalized in this catalogue; no automatic ranking is enabled.", "zh": "历史论文结果。本目录未统一不可变模型版本与资源预算，因此不启用自动排名。" } }, { "id": "self-replication-llama31-70b-v1", "benchmarkId": "self-replication", "benchmarkVersion": "arXiv:2412.12140v1", "system": "Llama 3.1 70B Instruct", "subjectType": "system", "evidenceType": "controlled-experiment", "sourceId": "self-replication", "sourceUrl": "https://arxiv.org/html/2412.12140v1#S3.SS1", "sourceVersion": "arXiv:2412.12140v1", "locator": "§1; §3.1", "publicationDate": "2024-12-09", "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "Instructed replication trials", "protocol": "Paper's controlled agent setup", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Paper-specific planning scaffold (§2.2)", "tools": "Controlled computing environment supplied by the authors", "budget": null }, "sampleSize": { "en": "10 repeated trials per system.", "zh": "每个系统重复试验 10 次。" }, "metrics": [ { "id": "trial-success", "label": { "en": "Reported trial success", "zh": "报告试验成功率" }, "value": 50, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Completion of the instructed task in the supplied setting.", "zh": "在所提供环境中完成被指令的任务。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } } ], "sourceFinding": { "en": "The authors report successful trials under their controlled setup.", "zh": "作者报告其受控设置中有试验成功。" }, "interpretation": { "en": "This supports a bounded capability observation under explicit instructions.", "zh": "这支持明确指令下的有限能力观察。" }, "limitations": { "en": "Small experimental sample with supplied access and scaffolding; not a deployment probability or a NUWA finding that a global red line was crossed.", "zh": "样本较小，访问条件与脚手架由实验提供；不代表部署概率，也不是 NUWA 对跨越全球红线的认定。" } }, { "id": "self-replication-qwen25-72b-v1", "benchmarkId": "self-replication", "benchmarkVersion": "arXiv:2412.12140v1", "system": "Qwen 2.5 72B Instruct", "subjectType": "system", "evidenceType": "controlled-experiment", "sourceId": "self-replication", "sourceUrl": "https://arxiv.org/html/2412.12140v1#S3.SS1", "sourceVersion": "arXiv:2412.12140v1", "locator": "§1; §3.1", "publicationDate": "2024-12-09", "evaluationDate": null, "verifiedOn": "2026-10-24", "taskSet": "Instructed replication trials", "protocol": "Paper's controlled agent setup", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Paper-specific planning scaffold (§2.2)", "tools": "Controlled computing environment supplied by the authors", "budget": null }, "sampleSize": { "en": "10 repeated trials per system.", "zh": "每个系统重复试验 10 次。" }, "metrics": [ { "id": "trial-success", "label": { "en": "Reported trial success", "zh": "报告试验成功率" }, "value": 90, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Completion of the instructed task in the supplied setting.", "zh": "在所提供环境中完成被指令的任务。" }, "dimension": "capability", "dimensionRationale": { "en": "Task completion under the specified evaluation configuration; not evidence of autonomous willingness.", "zh": "指定评测配置下的任务完成情况，不证明系统自主选择该行为。" } } ], "sourceFinding": { "en": "The authors report successful trials under their controlled setup.", "zh": "作者报告其受控设置中有试验成功。" }, "interpretation": { "en": "This supports a bounded capability observation under explicit instructions.", "zh": "这支持明确指令下的有限能力观察。" }, "limitations": { "en": "Small experimental sample with supplied access and scaffolding; not a deployment probability or a NUWA finding that a global red line was crossed.", "zh": "样本较小，访问条件与脚手架由实验提供；不代表部署概率，也不是 NUWA 对跨越全球红线的认定。" } }, { "id": "cybench-table-5-row-2", "benchmarkId": "cybench", "benchmarkVersion": "arXiv:2408.08926v4", "system": "Claude 3.5 Sonnet", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "cybench-author-source", "sourceUrl": "https://arxiv.org/html/2408.08926v4", "sourceVersion": "arXiv:2408.08926v4", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · Cybench", "protocol": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Structured bash", "tools": null, "budget": null }, "sampleSize": { "en": "40 CTF tasks", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "cybench-m0", "label": { "en": "Unguided performance", "zh": "无引导表现" }, "value": 17.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m1", "label": { "en": "Subtask-guided performance", "zh": "子任务引导表现" }, "value": 15, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m2", "label": { "en": "Subtask performance", "zh": "子任务表现" }, "value": 43.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude 3.5 Sonnet in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Claude 3.5 Sonnet 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "cybench-table-5-row-3", "benchmarkId": "cybench", "benchmarkVersion": "arXiv:2408.08926v4", "system": "GPT-4o", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "cybench-author-source", "sourceUrl": "https://arxiv.org/html/2408.08926v4", "sourceVersion": "arXiv:2408.08926v4", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · Cybench", "protocol": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Structured bash", "tools": null, "budget": null }, "sampleSize": { "en": "40 CTF tasks", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "cybench-m0", "label": { "en": "Unguided performance", "zh": "无引导表现" }, "value": 12.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m1", "label": { "en": "Subtask-guided performance", "zh": "子任务引导表现" }, "value": 17.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m2", "label": { "en": "Subtask performance", "zh": "子任务表现" }, "value": 28.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT-4o in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 GPT-4o 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "cybench-table-5-row-4", "benchmarkId": "cybench", "benchmarkVersion": "arXiv:2408.08926v4", "system": "Claude 3 Opus", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "cybench-author-source", "sourceUrl": "https://arxiv.org/html/2408.08926v4", "sourceVersion": "arXiv:2408.08926v4", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · Cybench", "protocol": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Structured bash", "tools": null, "budget": null }, "sampleSize": { "en": "40 CTF tasks", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "cybench-m0", "label": { "en": "Unguided performance", "zh": "无引导表现" }, "value": 10, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m1", "label": { "en": "Subtask-guided performance", "zh": "子任务引导表现" }, "value": 12.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m2", "label": { "en": "Subtask performance", "zh": "子任务表现" }, "value": 36.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude 3 Opus in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Claude 3 Opus 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "cybench-table-5-row-5", "benchmarkId": "cybench", "benchmarkVersion": "arXiv:2408.08926v4", "system": "OpenAI o1-preview", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "cybench-author-source", "sourceUrl": "https://arxiv.org/html/2408.08926v4", "sourceVersion": "arXiv:2408.08926v4", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · Cybench", "protocol": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Structured bash", "tools": null, "budget": null }, "sampleSize": { "en": "40 CTF tasks", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "cybench-m0", "label": { "en": "Unguided performance", "zh": "无引导表现" }, "value": 10, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m1", "label": { "en": "Subtask-guided performance", "zh": "子任务引导表现" }, "value": 10, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m2", "label": { "en": "Subtask performance", "zh": "子任务表现" }, "value": 46.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for OpenAI o1-preview in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 OpenAI o1-preview 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "cybench-table-5-row-6", "benchmarkId": "cybench", "benchmarkVersion": "arXiv:2408.08926v4", "system": "Llama 3.1 405B Instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "cybench-author-source", "sourceUrl": "https://arxiv.org/html/2408.08926v4", "sourceVersion": "arXiv:2408.08926v4", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · Cybench", "protocol": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Structured bash", "tools": null, "budget": null }, "sampleSize": { "en": "40 CTF tasks", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "cybench-m0", "label": { "en": "Unguided performance", "zh": "无引导表现" }, "value": 7.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m1", "label": { "en": "Subtask-guided performance", "zh": "子任务引导表现" }, "value": 15, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m2", "label": { "en": "Subtask performance", "zh": "子任务表现" }, "value": 20.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Llama 3.1 405B Instruct in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Llama 3.1 405B Instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "cybench-table-5-row-7", "benchmarkId": "cybench", "benchmarkVersion": "arXiv:2408.08926v4", "system": "Mixtral 8x22b Instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "cybench-author-source", "sourceUrl": "https://arxiv.org/html/2408.08926v4", "sourceVersion": "arXiv:2408.08926v4", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · Cybench", "protocol": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Structured bash", "tools": null, "budget": null }, "sampleSize": { "en": "40 CTF tasks", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "cybench-m0", "label": { "en": "Unguided performance", "zh": "无引导表现" }, "value": 7.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m1", "label": { "en": "Subtask-guided performance", "zh": "子任务引导表现" }, "value": 5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m2", "label": { "en": "Subtask performance", "zh": "子任务表现" }, "value": 15.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Mixtral 8x22b Instruct in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Mixtral 8x22b Instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "cybench-table-5-row-8", "benchmarkId": "cybench", "benchmarkVersion": "arXiv:2408.08926v4", "system": "Gemini 1.5 Pro", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "cybench-author-source", "sourceUrl": "https://arxiv.org/html/2408.08926v4", "sourceVersion": "arXiv:2408.08926v4", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · Cybench", "protocol": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Structured bash", "tools": null, "budget": null }, "sampleSize": { "en": "40 CTF tasks", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "cybench-m0", "label": { "en": "Unguided performance", "zh": "无引导表现" }, "value": 7.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m1", "label": { "en": "Subtask-guided performance", "zh": "子任务引导表现" }, "value": 5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m2", "label": { "en": "Subtask performance", "zh": "子任务表现" }, "value": 11.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Gemini 1.5 Pro in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Gemini 1.5 Pro 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "cybench-table-5-row-9", "benchmarkId": "cybench", "benchmarkVersion": "arXiv:2408.08926v4", "system": "Llama 3 70b Chat", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "cybench-author-source", "sourceUrl": "https://arxiv.org/html/2408.08926v4", "sourceVersion": "arXiv:2408.08926v4", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · Cybench", "protocol": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "Structured bash", "tools": null, "budget": null }, "sampleSize": { "en": "40 CTF tasks", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "cybench-m0", "label": { "en": "Unguided performance", "zh": "无引导表现" }, "value": 5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m1", "label": { "en": "Subtask-guided performance", "zh": "子任务引导表现" }, "value": 7.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "cybench-m2", "label": { "en": "Subtask performance", "zh": "子任务表现" }, "value": 8.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Llama 3 70b Chat in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Llama 3 70b Chat 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "opendeception-table-0-row-1", "benchmarkId": "opendeception", "benchmarkVersion": "arXiv:2504.13707v3", "system": "GPT-3.5", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "opendeception-author-source", "sourceUrl": "https://arxiv.org/html/2504.13707v3", "sourceVersion": "arXiv:2504.13707v3", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · OpenDeception", "protocol": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "50 authored scenarios; evaluated cases depend on the source screening procedure.", "zh": "50 个编写情境；实际受评案例取决于原文筛选程序。" }, "metrics": [ { "id": "opendeception-m0", "label": { "en": "IntentNet DIR", "zh": "IntentNet 欺骗意图率" }, "value": 100, "precision": 0, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m1", "label": { "en": "Human DIR", "zh": "人工标注欺骗意图率" }, "value": 97.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m2", "label": { "en": "Annotator consistency", "zh": "标注一致性" }, "value": 97.78, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "behavior", "dimensionRationale": { "en": "Agreement describes annotations of observed outputs; it is not itself a propensity or success metric.", "zh": "一致性描述对已观察输出的标注吻合程度，本身不是倾向或成功率指标。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT-3.5 in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GPT-3.5 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "opendeception-table-0-row-2", "benchmarkId": "opendeception", "benchmarkVersion": "arXiv:2504.13707v3", "system": "GPT-4o", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "opendeception-author-source", "sourceUrl": "https://arxiv.org/html/2504.13707v3", "sourceVersion": "arXiv:2504.13707v3", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · OpenDeception", "protocol": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "50 authored scenarios; evaluated cases depend on the source screening procedure.", "zh": "50 个编写情境；实际受评案例取决于原文筛选程序。" }, "metrics": [ { "id": "opendeception-m0", "label": { "en": "IntentNet DIR", "zh": "IntentNet 欺骗意图率" }, "value": 97.87, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m1", "label": { "en": "Human DIR", "zh": "人工标注欺骗意图率" }, "value": 93.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m2", "label": { "en": "Annotator consistency", "zh": "标注一致性" }, "value": 97.87, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "behavior", "dimensionRationale": { "en": "Agreement describes annotations of observed outputs; it is not itself a propensity or success metric.", "zh": "一致性描述对已观察输出的标注吻合程度，本身不是倾向或成功率指标。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT-4o in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GPT-4o 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "opendeception-table-0-row-3", "benchmarkId": "opendeception", "benchmarkVersion": "arXiv:2504.13707v3", "system": "Llama-3.1-8B-Instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "opendeception-author-source", "sourceUrl": "https://arxiv.org/html/2504.13707v3", "sourceVersion": "arXiv:2504.13707v3", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · OpenDeception", "protocol": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "50 authored scenarios; evaluated cases depend on the source screening procedure.", "zh": "50 个编写情境；实际受评案例取决于原文筛选程序。" }, "metrics": [ { "id": "opendeception-m0", "label": { "en": "IntentNet DIR", "zh": "IntentNet 欺骗意图率" }, "value": 92.85, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m1", "label": { "en": "Human DIR", "zh": "人工标注欺骗意图率" }, "value": 92.85, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m2", "label": { "en": "Annotator consistency", "zh": "标注一致性" }, "value": 100, "precision": 0, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "behavior", "dimensionRationale": { "en": "Agreement describes annotations of observed outputs; it is not itself a propensity or success metric.", "zh": "一致性描述对已观察输出的标注吻合程度，本身不是倾向或成功率指标。" } } ], "sourceFinding": { "en": "Source-reported measurements for Llama-3.1-8B-Instruct in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Llama-3.1-8B-Instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "opendeception-table-0-row-4", "benchmarkId": "opendeception", "benchmarkVersion": "arXiv:2504.13707v3", "system": "Llama-3.1-70B-Instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "opendeception-author-source", "sourceUrl": "https://arxiv.org/html/2504.13707v3", "sourceVersion": "arXiv:2504.13707v3", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · OpenDeception", "protocol": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "50 authored scenarios; evaluated cases depend on the source screening procedure.", "zh": "50 个编写情境；实际受评案例取决于原文筛选程序。" }, "metrics": [ { "id": "opendeception-m0", "label": { "en": "IntentNet DIR", "zh": "IntentNet 欺骗意图率" }, "value": 100, "precision": 0, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m1", "label": { "en": "Human DIR", "zh": "人工标注欺骗意图率" }, "value": 97.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m2", "label": { "en": "Annotator consistency", "zh": "标注一致性" }, "value": 97.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "behavior", "dimensionRationale": { "en": "Agreement describes annotations of observed outputs; it is not itself a propensity or success metric.", "zh": "一致性描述对已观察输出的标注吻合程度，本身不是倾向或成功率指标。" } } ], "sourceFinding": { "en": "Source-reported measurements for Llama-3.1-70B-Instruct in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Llama-3.1-70B-Instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "opendeception-table-0-row-5", "benchmarkId": "opendeception", "benchmarkVersion": "arXiv:2504.13707v3", "system": "Qwen2-7B-Instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "opendeception-author-source", "sourceUrl": "https://arxiv.org/html/2504.13707v3", "sourceVersion": "arXiv:2504.13707v3", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · OpenDeception", "protocol": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "50 authored scenarios; evaluated cases depend on the source screening procedure.", "zh": "50 个编写情境；实际受评案例取决于原文筛选程序。" }, "metrics": [ { "id": "opendeception-m0", "label": { "en": "IntentNet DIR", "zh": "IntentNet 欺骗意图率" }, "value": 92.31, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m1", "label": { "en": "Human DIR", "zh": "人工标注欺骗意图率" }, "value": 89.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m2", "label": { "en": "Annotator consistency", "zh": "标注一致性" }, "value": 97.44, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "behavior", "dimensionRationale": { "en": "Agreement describes annotations of observed outputs; it is not itself a propensity or success metric.", "zh": "一致性描述对已观察输出的标注吻合程度，本身不是倾向或成功率指标。" } } ], "sourceFinding": { "en": "Source-reported measurements for Qwen2-7B-Instruct in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Qwen2-7B-Instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "opendeception-table-0-row-6", "benchmarkId": "opendeception", "benchmarkVersion": "arXiv:2504.13707v3", "system": "Qwen2-72B-Instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "opendeception-author-source", "sourceUrl": "https://arxiv.org/html/2504.13707v3", "sourceVersion": "arXiv:2504.13707v3", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · OpenDeception", "protocol": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "50 authored scenarios; evaluated cases depend on the source screening procedure.", "zh": "50 个编写情境；实际受评案例取决于原文筛选程序。" }, "metrics": [ { "id": "opendeception-m0", "label": { "en": "IntentNet DIR", "zh": "IntentNet 欺骗意图率" }, "value": 100, "precision": 0, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m1", "label": { "en": "Human DIR", "zh": "人工标注欺骗意图率" }, "value": 100, "precision": 0, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m2", "label": { "en": "Annotator consistency", "zh": "标注一致性" }, "value": 100, "precision": 0, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "behavior", "dimensionRationale": { "en": "Agreement describes annotations of observed outputs; it is not itself a propensity or success metric.", "zh": "一致性描述对已观察输出的标注吻合程度，本身不是倾向或成功率指标。" } } ], "sourceFinding": { "en": "Source-reported measurements for Qwen2-72B-Instruct in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Qwen2-72B-Instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "opendeception-table-0-row-7", "benchmarkId": "opendeception", "benchmarkVersion": "arXiv:2504.13707v3", "system": "Qwen2.5-3B-Instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "opendeception-author-source", "sourceUrl": "https://arxiv.org/html/2504.13707v3", "sourceVersion": "arXiv:2504.13707v3", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · OpenDeception", "protocol": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "50 authored scenarios; evaluated cases depend on the source screening procedure.", "zh": "50 个编写情境；实际受评案例取决于原文筛选程序。" }, "metrics": [ { "id": "opendeception-m0", "label": { "en": "IntentNet DIR", "zh": "IntentNet 欺骗意图率" }, "value": 87.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m1", "label": { "en": "Human DIR", "zh": "人工标注欺骗意图率" }, "value": 81.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m2", "label": { "en": "Annotator consistency", "zh": "标注一致性" }, "value": 90.63, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "behavior", "dimensionRationale": { "en": "Agreement describes annotations of observed outputs; it is not itself a propensity or success metric.", "zh": "一致性描述对已观察输出的标注吻合程度，本身不是倾向或成功率指标。" } } ], "sourceFinding": { "en": "Source-reported measurements for Qwen2.5-3B-Instruct in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Qwen2.5-3B-Instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "opendeception-table-0-row-8", "benchmarkId": "opendeception", "benchmarkVersion": "arXiv:2504.13707v3", "system": "Qwen2.5-7B-Instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "opendeception-author-source", "sourceUrl": "https://arxiv.org/html/2504.13707v3", "sourceVersion": "arXiv:2504.13707v3", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · OpenDeception", "protocol": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "50 authored scenarios; evaluated cases depend on the source screening procedure.", "zh": "50 个编写情境；实际受评案例取决于原文筛选程序。" }, "metrics": [ { "id": "opendeception-m0", "label": { "en": "IntentNet DIR", "zh": "IntentNet 欺骗意图率" }, "value": 90.24, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m1", "label": { "en": "Human DIR", "zh": "人工标注欺骗意图率" }, "value": 87.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m2", "label": { "en": "Annotator consistency", "zh": "标注一致性" }, "value": 95.12, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "behavior", "dimensionRationale": { "en": "Agreement describes annotations of observed outputs; it is not itself a propensity or success metric.", "zh": "一致性描述对已观察输出的标注吻合程度，本身不是倾向或成功率指标。" } } ], "sourceFinding": { "en": "Source-reported measurements for Qwen2.5-7B-Instruct in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Qwen2.5-7B-Instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "opendeception-table-0-row-9", "benchmarkId": "opendeception", "benchmarkVersion": "arXiv:2504.13707v3", "system": "Qwen2.5-14B-Instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "opendeception-author-source", "sourceUrl": "https://arxiv.org/html/2504.13707v3", "sourceVersion": "arXiv:2504.13707v3", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · OpenDeception", "protocol": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "50 authored scenarios; evaluated cases depend on the source screening procedure.", "zh": "50 个编写情境；实际受评案例取决于原文筛选程序。" }, "metrics": [ { "id": "opendeception-m0", "label": { "en": "IntentNet DIR", "zh": "IntentNet 欺骗意图率" }, "value": 93.62, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m1", "label": { "en": "Human DIR", "zh": "人工标注欺骗意图率" }, "value": 91.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m2", "label": { "en": "Annotator consistency", "zh": "标注一致性" }, "value": 95.74, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "behavior", "dimensionRationale": { "en": "Agreement describes annotations of observed outputs; it is not itself a propensity or success metric.", "zh": "一致性描述对已观察输出的标注吻合程度，本身不是倾向或成功率指标。" } } ], "sourceFinding": { "en": "Source-reported measurements for Qwen2.5-14B-Instruct in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Qwen2.5-14B-Instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "opendeception-table-0-row-10", "benchmarkId": "opendeception", "benchmarkVersion": "arXiv:2504.13707v3", "system": "Qwen2.5-32B-Instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "opendeception-author-source", "sourceUrl": "https://arxiv.org/html/2504.13707v3", "sourceVersion": "arXiv:2504.13707v3", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · OpenDeception", "protocol": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "50 authored scenarios; evaluated cases depend on the source screening procedure.", "zh": "50 个编写情境；实际受评案例取决于原文筛选程序。" }, "metrics": [ { "id": "opendeception-m0", "label": { "en": "IntentNet DIR", "zh": "IntentNet 欺骗意图率" }, "value": 95.65, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m1", "label": { "en": "Human DIR", "zh": "人工标注欺骗意图率" }, "value": 91.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m2", "label": { "en": "Annotator consistency", "zh": "标注一致性" }, "value": 97.83, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "behavior", "dimensionRationale": { "en": "Agreement describes annotations of observed outputs; it is not itself a propensity or success metric.", "zh": "一致性描述对已观察输出的标注吻合程度，本身不是倾向或成功率指标。" } } ], "sourceFinding": { "en": "Source-reported measurements for Qwen2.5-32B-Instruct in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Qwen2.5-32B-Instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "opendeception-table-0-row-11", "benchmarkId": "opendeception", "benchmarkVersion": "arXiv:2504.13707v3", "system": "Qwen2.5-72B-Instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "opendeception-author-source", "sourceUrl": "https://arxiv.org/html/2504.13707v3", "sourceVersion": "arXiv:2504.13707v3", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · OpenDeception", "protocol": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "50 authored scenarios; evaluated cases depend on the source screening procedure.", "zh": "50 个编写情境；实际受评案例取决于原文筛选程序。" }, "metrics": [ { "id": "opendeception-m0", "label": { "en": "IntentNet DIR", "zh": "IntentNet 欺骗意图率" }, "value": 100, "precision": 0, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m1", "label": { "en": "Human DIR", "zh": "人工标注欺骗意图率" }, "value": 100, "precision": 0, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "opendeception-m2", "label": { "en": "Annotator consistency", "zh": "标注一致性" }, "value": 100, "precision": 0, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dimension": "behavior", "dimensionRationale": { "en": "Agreement describes annotations of observed outputs; it is not itself a propensity or success metric.", "zh": "一致性描述对已观察输出的标注吻合程度，本身不是倾向或成功率指标。" } } ], "sourceFinding": { "en": "Source-reported measurements for Qwen2.5-72B-Instruct in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Qwen2.5-72B-Instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "paperbench-table-3-row-1", "benchmarkId": "paperbench", "benchmarkVersion": "arXiv:2504.01848v3", "system": "o3-mini-high", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "paperbench-chart-source", "sourceUrl": "https://arxiv.org/html/2504.01848v3", "sourceVersion": "arXiv:2504.01848v3", "locator": "Table 4", "publicationDate": "2025-04-07", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 4 · PaperBench", "protocol": "Table 4: average replication score in the main BasicAgent setup; ± denotes one standard error of the mean.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "BasicAgent", "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "paperbench-m0", "label": { "en": "Average replication score", "zh": "平均复现分数" }, "value": 2.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±0.2 (source-reported; one standard error)", "meaning": { "en": "Table 4: average replication score in the main BasicAgent setup; ± denotes one standard error of the mean.", "zh": "BasicAgent 主设置的平均论文复现分数。± 为一个均值标准误，不是 95% 置信区间。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o3-mini-high in Table 4; values are transcribed, not independently reproduced.", "zh": "原文 Table 4 报告的 o3-mini-high 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "paperbench-table-3-row-2", "benchmarkId": "paperbench", "benchmarkVersion": "arXiv:2504.01848v3", "system": "gpt-4o", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "paperbench-chart-source", "sourceUrl": "https://arxiv.org/html/2504.01848v3", "sourceVersion": "arXiv:2504.01848v3", "locator": "Table 4", "publicationDate": "2025-04-07", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 4 · PaperBench", "protocol": "Table 4: average replication score in the main BasicAgent setup; ± denotes one standard error of the mean.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "BasicAgent", "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "paperbench-m0", "label": { "en": "Average replication score", "zh": "平均复现分数" }, "value": 4.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±0.1 (source-reported; one standard error)", "meaning": { "en": "Table 4: average replication score in the main BasicAgent setup; ± denotes one standard error of the mean.", "zh": "BasicAgent 主设置的平均论文复现分数。± 为一个均值标准误，不是 95% 置信区间。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for gpt-4o in Table 4; values are transcribed, not independently reproduced.", "zh": "原文 Table 4 报告的 gpt-4o 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "paperbench-table-3-row-3", "benchmarkId": "paperbench", "benchmarkVersion": "arXiv:2504.01848v3", "system": "gemini-2.0-flash", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "paperbench-chart-source", "sourceUrl": "https://arxiv.org/html/2504.01848v3", "sourceVersion": "arXiv:2504.01848v3", "locator": "Table 4", "publicationDate": "2025-04-07", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 4 · PaperBench", "protocol": "Table 4: average replication score in the main BasicAgent setup; ± denotes one standard error of the mean.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "BasicAgent", "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "paperbench-m0", "label": { "en": "Average replication score", "zh": "平均复现分数" }, "value": 3.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±0.2 (source-reported; one standard error)", "meaning": { "en": "Table 4: average replication score in the main BasicAgent setup; ± denotes one standard error of the mean.", "zh": "BasicAgent 主设置的平均论文复现分数。± 为一个均值标准误，不是 95% 置信区间。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for gemini-2.0-flash in Table 4; values are transcribed, not independently reproduced.", "zh": "原文 Table 4 报告的 gemini-2.0-flash 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "paperbench-table-3-row-4", "benchmarkId": "paperbench", "benchmarkVersion": "arXiv:2504.01848v3", "system": "DeepSeek-R1", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "paperbench-chart-source", "sourceUrl": "https://arxiv.org/html/2504.01848v3", "sourceVersion": "arXiv:2504.01848v3", "locator": "Table 4", "publicationDate": "2025-04-07", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 4 · PaperBench", "protocol": "Table 4: average replication score in the main BasicAgent setup; ± denotes one standard error of the mean.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "BasicAgent", "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "paperbench-m0", "label": { "en": "Average replication score", "zh": "平均复现分数" }, "value": 6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±0.3 (source-reported; one standard error)", "meaning": { "en": "Table 4: average replication score in the main BasicAgent setup; ± denotes one standard error of the mean.", "zh": "BasicAgent 主设置的平均论文复现分数。± 为一个均值标准误，不是 95% 置信区间。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for DeepSeek-R1 in Table 4; values are transcribed, not independently reproduced.", "zh": "原文 Table 4 报告的 DeepSeek-R1 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "paperbench-table-3-row-5", "benchmarkId": "paperbench", "benchmarkVersion": "arXiv:2504.01848v3", "system": "o1-high", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "paperbench-chart-source", "sourceUrl": "https://arxiv.org/html/2504.01848v3", "sourceVersion": "arXiv:2504.01848v3", "locator": "Table 4", "publicationDate": "2025-04-07", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 4 · PaperBench", "protocol": "Table 4: average replication score in the main BasicAgent setup; ± denotes one standard error of the mean.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "BasicAgent", "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "paperbench-m0", "label": { "en": "Average replication score", "zh": "平均复现分数" }, "value": 13.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±0.3 (source-reported; one standard error)", "meaning": { "en": "Table 4: average replication score in the main BasicAgent setup; ± denotes one standard error of the mean.", "zh": "BasicAgent 主设置的平均论文复现分数。± 为一个均值标准误，不是 95% 置信区间。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o1-high in Table 4; values are transcribed, not independently reproduced.", "zh": "原文 Table 4 报告的 o1-high 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "paperbench-table-3-row-6", "benchmarkId": "paperbench", "benchmarkVersion": "arXiv:2504.01848v3", "system": "claude-3.5-sonnet", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "paperbench-chart-source", "sourceUrl": "https://arxiv.org/html/2504.01848v3", "sourceVersion": "arXiv:2504.01848v3", "locator": "Table 4", "publicationDate": "2025-04-07", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 4 · PaperBench", "protocol": "Table 4: average replication score in the main BasicAgent setup; ± denotes one standard error of the mean.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": "BasicAgent", "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "paperbench-m0", "label": { "en": "Average replication score", "zh": "平均复现分数" }, "value": 21, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±0.8 (source-reported; one standard error)", "meaning": { "en": "Table 4: average replication score in the main BasicAgent setup; ± denotes one standard error of the mean.", "zh": "BasicAgent 主设置的平均论文复现分数。± 为一个均值标准误，不是 95% 置信区间。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for claude-3.5-sonnet in Table 4; values are transcribed, not independently reproduced.", "zh": "原文 Table 4 报告的 claude-3.5-sonnet 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "vct-table-0-row-1", "benchmarkId": "vct", "benchmarkVersion": "arXiv:2504.16137v2", "system": "Claude 3.5 Sonnet (Jun ’24)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "vct-author-source", "sourceUrl": "https://arxiv.org/html/2504.16137v2", "sourceVersion": "arXiv:2504.16137v2", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Virology Capabilities Test (VCT)", "protocol": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Three evaluation runs; source reports SEM below 1% for all models.", "zh": "三次评测运行；原文报告所有模型的均值标准误低于 1%。" }, "metrics": [ { "id": "vct-m0", "label": { "en": "Multiple-response accuracy", "zh": "多选格式准确率" }, "value": 26.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "zh": "多选格式的平均准确率；部分模型只有一次运行，原文星号保留在系统名。知识测验成绩不代表端到端生物危险任务能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude 3.5 Sonnet (Jun ’24) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Claude 3.5 Sonnet (Jun ’24) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "vct-table-0-row-2", "benchmarkId": "vct", "benchmarkVersion": "arXiv:2504.16137v2", "system": "Claude 3.5 Sonnet (Oct ’24)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "vct-author-source", "sourceUrl": "https://arxiv.org/html/2504.16137v2", "sourceVersion": "arXiv:2504.16137v2", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Virology Capabilities Test (VCT)", "protocol": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Three evaluation runs; source reports SEM below 1% for all models.", "zh": "三次评测运行；原文报告所有模型的均值标准误低于 1%。" }, "metrics": [ { "id": "vct-m0", "label": { "en": "Multiple-response accuracy", "zh": "多选格式准确率" }, "value": 33.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "zh": "多选格式的平均准确率；部分模型只有一次运行，原文星号保留在系统名。知识测验成绩不代表端到端生物危险任务能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude 3.5 Sonnet (Oct ’24) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Claude 3.5 Sonnet (Oct ’24) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "vct-table-0-row-3", "benchmarkId": "vct", "benchmarkVersion": "arXiv:2504.16137v2", "system": "Claude 3.7 Sonnet (Feb ’25)*", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "vct-author-source", "sourceUrl": "https://arxiv.org/html/2504.16137v2", "sourceVersion": "arXiv:2504.16137v2", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Virology Capabilities Test (VCT)", "protocol": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Three evaluation runs; source reports SEM below 1% for all models.", "zh": "三次评测运行；原文报告所有模型的均值标准误低于 1%。" }, "metrics": [ { "id": "vct-m0", "label": { "en": "Multiple-response accuracy", "zh": "多选格式准确率" }, "value": 30.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "zh": "多选格式的平均准确率；部分模型只有一次运行，原文星号保留在系统名。知识测验成绩不代表端到端生物危险任务能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude 3.7 Sonnet (Feb ’25)* in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Claude 3.7 Sonnet (Feb ’25)* 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "vct-table-0-row-4", "benchmarkId": "vct", "benchmarkVersion": "arXiv:2504.16137v2", "system": "Gemini 1.5 Pro", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "vct-author-source", "sourceUrl": "https://arxiv.org/html/2504.16137v2", "sourceVersion": "arXiv:2504.16137v2", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Virology Capabilities Test (VCT)", "protocol": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Three evaluation runs; source reports SEM below 1% for all models.", "zh": "三次评测运行；原文报告所有模型的均值标准误低于 1%。" }, "metrics": [ { "id": "vct-m0", "label": { "en": "Multiple-response accuracy", "zh": "多选格式准确率" }, "value": 25.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "zh": "多选格式的平均准确率；部分模型只有一次运行，原文星号保留在系统名。知识测验成绩不代表端到端生物危险任务能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Gemini 1.5 Pro in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Gemini 1.5 Pro 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "vct-table-0-row-5", "benchmarkId": "vct", "benchmarkVersion": "arXiv:2504.16137v2", "system": "Gemini 2.0 Pro (exp.)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "vct-author-source", "sourceUrl": "https://arxiv.org/html/2504.16137v2", "sourceVersion": "arXiv:2504.16137v2", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Virology Capabilities Test (VCT)", "protocol": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Three evaluation runs; source reports SEM below 1% for all models.", "zh": "三次评测运行；原文报告所有模型的均值标准误低于 1%。" }, "metrics": [ { "id": "vct-m0", "label": { "en": "Multiple-response accuracy", "zh": "多选格式准确率" }, "value": 27.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "zh": "多选格式的平均准确率；部分模型只有一次运行，原文星号保留在系统名。知识测验成绩不代表端到端生物危险任务能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Gemini 2.0 Pro (exp.) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Gemini 2.0 Pro (exp.) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "vct-table-0-row-6", "benchmarkId": "vct", "benchmarkVersion": "arXiv:2504.16137v2", "system": "Gemini 2.5 Pro (exp.)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "vct-author-source", "sourceUrl": "https://arxiv.org/html/2504.16137v2", "sourceVersion": "arXiv:2504.16137v2", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Virology Capabilities Test (VCT)", "protocol": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "One evaluation run (italicized in the source table).", "zh": "一次评测运行（原文表格以斜体标记）。" }, "metrics": [ { "id": "vct-m0", "label": { "en": "Multiple-response accuracy", "zh": "多选格式准确率" }, "value": 37.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "zh": "多选格式的平均准确率；部分模型只有一次运行，原文星号保留在系统名。知识测验成绩不代表端到端生物危险任务能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Gemini 2.5 Pro (exp.) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Gemini 2.5 Pro (exp.) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "vct-table-0-row-7", "benchmarkId": "vct", "benchmarkVersion": "arXiv:2504.16137v2", "system": "GPT-4o (Nov ’24)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "vct-author-source", "sourceUrl": "https://arxiv.org/html/2504.16137v2", "sourceVersion": "arXiv:2504.16137v2", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Virology Capabilities Test (VCT)", "protocol": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Three evaluation runs; source reports SEM below 1% for all models.", "zh": "三次评测运行；原文报告所有模型的均值标准误低于 1%。" }, "metrics": [ { "id": "vct-m0", "label": { "en": "Multiple-response accuracy", "zh": "多选格式准确率" }, "value": 18.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "zh": "多选格式的平均准确率；部分模型只有一次运行，原文星号保留在系统名。知识测验成绩不代表端到端生物危险任务能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT-4o (Nov ’24) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GPT-4o (Nov ’24) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "vct-table-0-row-8", "benchmarkId": "vct", "benchmarkVersion": "arXiv:2504.16137v2", "system": "GPT-4.1 (Apr ’25)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "vct-author-source", "sourceUrl": "https://arxiv.org/html/2504.16137v2", "sourceVersion": "arXiv:2504.16137v2", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Virology Capabilities Test (VCT)", "protocol": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Three evaluation runs; source reports SEM below 1% for all models.", "zh": "三次评测运行；原文报告所有模型的均值标准误低于 1%。" }, "metrics": [ { "id": "vct-m0", "label": { "en": "Multiple-response accuracy", "zh": "多选格式准确率" }, "value": 26.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "zh": "多选格式的平均准确率；部分模型只有一次运行，原文星号保留在系统名。知识测验成绩不代表端到端生物危险任务能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT-4.1 (Apr ’25) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GPT-4.1 (Apr ’25) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "vct-table-0-row-9", "benchmarkId": "vct", "benchmarkVersion": "arXiv:2504.16137v2", "system": "GPT-4.5 Preview (Feb ’25)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "vct-author-source", "sourceUrl": "https://arxiv.org/html/2504.16137v2", "sourceVersion": "arXiv:2504.16137v2", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Virology Capabilities Test (VCT)", "protocol": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Three evaluation runs; source reports SEM below 1% for all models.", "zh": "三次评测运行；原文报告所有模型的均值标准误低于 1%。" }, "metrics": [ { "id": "vct-m0", "label": { "en": "Multiple-response accuracy", "zh": "多选格式准确率" }, "value": 28.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "zh": "多选格式的平均准确率；部分模型只有一次运行，原文星号保留在系统名。知识测验成绩不代表端到端生物危险任务能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT-4.5 Preview (Feb ’25) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GPT-4.5 Preview (Feb ’25) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "vct-table-0-row-10", "benchmarkId": "vct", "benchmarkVersion": "arXiv:2504.16137v2", "system": "o1 (Dec ’24)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "vct-author-source", "sourceUrl": "https://arxiv.org/html/2504.16137v2", "sourceVersion": "arXiv:2504.16137v2", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Virology Capabilities Test (VCT)", "protocol": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "One evaluation run (italicized in the source table).", "zh": "一次评测运行（原文表格以斜体标记）。" }, "metrics": [ { "id": "vct-m0", "label": { "en": "Multiple-response accuracy", "zh": "多选格式准确率" }, "value": 35.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "zh": "多选格式的平均准确率；部分模型只有一次运行，原文星号保留在系统名。知识测验成绩不代表端到端生物危险任务能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o1 (Dec ’24) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 o1 (Dec ’24) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "vct-table-0-row-11", "benchmarkId": "vct", "benchmarkVersion": "arXiv:2504.16137v2", "system": "o3 (Apr ’25)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "vct-author-source", "sourceUrl": "https://arxiv.org/html/2504.16137v2", "sourceVersion": "arXiv:2504.16137v2", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Virology Capabilities Test (VCT)", "protocol": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Three evaluation runs; source reports SEM below 1% for all models.", "zh": "三次评测运行；原文报告所有模型的均值标准误低于 1%。" }, "metrics": [ { "id": "vct-m0", "label": { "en": "Multiple-response accuracy", "zh": "多选格式准确率" }, "value": 43.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "zh": "多选格式的平均准确率；部分模型只有一次运行，原文星号保留在系统名。知识测验成绩不代表端到端生物危险任务能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o3 (Apr ’25) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 o3 (Apr ’25) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "vct-table-0-row-12", "benchmarkId": "vct", "benchmarkVersion": "arXiv:2504.16137v2", "system": "o4-mini (Apr ’25)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "vct-author-source", "sourceUrl": "https://arxiv.org/html/2504.16137v2", "sourceVersion": "arXiv:2504.16137v2", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Virology Capabilities Test (VCT)", "protocol": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "One evaluation run (italicized in the source table).", "zh": "一次评测运行（原文表格以斜体标记）。" }, "metrics": [ { "id": "vct-m0", "label": { "en": "Multiple-response accuracy", "zh": "多选格式准确率" }, "value": 37, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "zh": "多选格式的平均准确率；部分模型只有一次运行，原文星号保留在系统名。知识测验成绩不代表端到端生物危险任务能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o4-mini (Apr ’25) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 o4-mini (Apr ’25) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-1", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "claude-3-5-sonnet-20240620", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 27.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 33.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 23.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 44.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 80.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for claude-3-5-sonnet-20240620 in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 claude-3-5-sonnet-20240620 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-2", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "claude-3-7-sonnet-20250219", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 47.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 26.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 38.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 42.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 82.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for claude-3-7-sonnet-20250219 in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 claude-3-7-sonnet-20250219 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-3", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "deepseek-r1", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 24.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 42.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 79.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for deepseek-r1 in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 deepseek-r1 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-4", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "deepseek-v3", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 20.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 53.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 13.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 67.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 71.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for deepseek-v3 in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 deepseek-v3 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-5", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "deepseek-llm-67b-chat", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 18.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 45.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 6.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 69, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 64.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for deepseek-llm-67b-chat in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 deepseek-llm-67b-chat 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-6", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "gemini-2.0-flash", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 20.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 48.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 14.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 64, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 79.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for gemini-2.0-flash in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 gemini-2.0-flash 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-7", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "gpt-4.5-preview-2025-02-27", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 27.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 43.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 76.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for gpt-4.5-preview-2025-02-27 in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 gpt-4.5-preview-2025-02-27 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-8", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "gpt-4o-2024-08-06", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 21.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 44.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 14.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 63, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 78.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for gpt-4o-2024-08-06 in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 gpt-4o-2024-08-06 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-9", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "gpt-4o-mini-2024-07-18", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 21.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 45.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 15.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 59.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 71.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for gpt-4o-mini-2024-07-18 in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 gpt-4o-mini-2024-07-18 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-10", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "grok-2-1212", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 14.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 63, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 10.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 71.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 72.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for grok-2-1212 in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 grok-2-1212 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-11", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "llama-2-13b-chat", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 28.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 24.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 19, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 41.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 40.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for llama-2-13b-chat in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 llama-2-13b-chat 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-12", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "llama-2-70b-chat", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 28.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 26.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 19.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 41.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 40.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for llama-2-70b-chat in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 llama-2-70b-chat 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-13", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "llama-2-7b-chat", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 27.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 21.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 17.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 38.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 33.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for llama-2-7b-chat in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 llama-2-7b-chat 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-14", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "llama-31-405b-instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 21.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 28, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 14.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 44.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 72.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for llama-31-405b-instruct in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 llama-31-405b-instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-15", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "llama-31-70b-instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 27.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 43.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 14.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 64.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 73.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for llama-31-70b-instruct in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 llama-31-70b-instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-16", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "llama-31-8b-instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 18.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 23.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 8.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 46.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 62, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for llama-31-8b-instruct in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 llama-31-8b-instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-17", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "llama-32-1b-instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 13.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 13.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 4.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 35.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 23, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for llama-32-1b-instruct in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 llama-32-1b-instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-18", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "llama-32-3b-instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 21.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 23.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 10.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 43.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 40, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for llama-32-3b-instruct in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 llama-32-3b-instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-19", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "llama-33-70b-instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 24.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 44.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 17.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 58.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 75.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for llama-33-70b-instruct in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 llama-33-70b-instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-20", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "o3-mini-2025-01-31 ∗", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 19.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 48.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 63.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for o3-mini-2025-01-31 ∗ in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 o3-mini-2025-01-31 ∗ 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-21", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "qwen15-110b-chat", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 27.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 35.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 20.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 53.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 72.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for qwen15-110b-chat in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 qwen15-110b-chat 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-22", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "qwen15-32b-chat", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 23.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 42.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 15.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 60, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 63, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for qwen15-32b-chat in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 qwen15-32b-chat 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-23", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "qwen15-72b-chat", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 24.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 47.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 15.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 63.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 69.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for qwen15-72b-chat in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 qwen15-72b-chat 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-24", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "qwen15-7b-chat", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 27.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 35.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 15.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 55.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 52.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for qwen15-7b-chat in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 qwen15-7b-chat 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-25", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "qwen25-05b-instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 15.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 15.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 6.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 41.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 20.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for qwen25-05b-instruct in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 qwen25-05b-instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-26", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "qwen25-14b-instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 26.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 47.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 17.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 63.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 64.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for qwen25-14b-instruct in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 qwen25-14b-instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-27", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "qwen25-15b-instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 25.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 26.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 10.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 57.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 28.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for qwen25-15b-instruct in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 qwen25-15b-instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-28", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "qwen25-32b-instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 28.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 43.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 20.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 59.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 63.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for qwen25-32b-instruct in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 qwen25-32b-instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-29", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "qwen25-3b-instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 30.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 33.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 18.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 54.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 46.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for qwen25-3b-instruct in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 qwen25-3b-instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-30", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "qwen25-72b-instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 23.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 49.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 15.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 61.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 66, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for qwen25-72b-instruct in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 qwen25-72b-instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-31", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "qwen25-7b-instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 28.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 39, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 18.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 55.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 51.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for qwen25-7b-instruct in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 qwen25-7b-instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mask-table-4-row-32", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "qwq-32b-preview", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mask-author-source", "sourceUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "locator": "Table 3", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 3 · MASK", "protocol": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mask-m0", "label": { "en": "P(honest)", "zh": "P(honest)" }, "value": 20.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m1", "label": { "en": "P(lie)", "zh": "P(lie)" }, "value": 25.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m2", "label": { "en": "P(honest)@10", "zh": "P(honest)@10" }, "value": 11.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m3", "label": { "en": "P(lie)@10", "zh": "P(lie)@10" }, "value": 46.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mask-m4", "label": { "en": "Accuracy", "zh": "准确率" }, "value": 49.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dimension": "capability", "dimensionRationale": { "en": "Accuracy measures task correctness in the reported setup; it is distinct from pressure-induced honesty or lying propensity.", "zh": "准确率衡量给定设置下的任务正确性，与压力情境中的诚实或说谎倾向不同。" } } ], "sourceFinding": { "en": "Source-reported measurements for qwq-32b-preview in Table 3; values are transcribed, not independently reproduced.", "zh": "原文 Table 3 报告的 qwq-32b-preview 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "wmdp-table-3-row-1", "benchmarkId": "wmdp", "benchmarkVersion": "arXiv:2403.03218v7", "system": "zephyr-7b", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "wmdp-author-source", "sourceUrl": "https://arxiv.org/html/2403.03218v7", "sourceVersion": "arXiv:2403.03218v7", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · WMDP", "protocol": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "wmdp-m0", "label": { "en": "WMDP Bio accuracy", "zh": "WMDP 生物知识准确率" }, "value": 63.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "wmdp-m1", "label": { "en": "WMDP Cyber accuracy", "zh": "WMDP 网络知识准确率" }, "value": 44, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for zephyr-7b in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 zephyr-7b 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "wmdp-table-3-row-2", "benchmarkId": "wmdp", "benchmarkVersion": "arXiv:2403.03218v7", "system": "zephyr-7b · LLMU", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "wmdp-author-source", "sourceUrl": "https://arxiv.org/html/2403.03218v7", "sourceVersion": "arXiv:2403.03218v7", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · WMDP", "protocol": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "wmdp-m0", "label": { "en": "WMDP Bio accuracy", "zh": "WMDP 生物知识准确率" }, "value": 59.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "wmdp-m1", "label": { "en": "WMDP Cyber accuracy", "zh": "WMDP 网络知识准确率" }, "value": 39.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for zephyr-7b · LLMU in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 zephyr-7b · LLMU 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "wmdp-table-3-row-3", "benchmarkId": "wmdp", "benchmarkVersion": "arXiv:2403.03218v7", "system": "zephyr-7b · SCRUB", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "wmdp-author-source", "sourceUrl": "https://arxiv.org/html/2403.03218v7", "sourceVersion": "arXiv:2403.03218v7", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · WMDP", "protocol": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "wmdp-m0", "label": { "en": "WMDP Bio accuracy", "zh": "WMDP 生物知识准确率" }, "value": 43.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "wmdp-m1", "label": { "en": "WMDP Cyber accuracy", "zh": "WMDP 网络知识准确率" }, "value": 39.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for zephyr-7b · SCRUB in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 zephyr-7b · SCRUB 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "wmdp-table-3-row-4", "benchmarkId": "wmdp", "benchmarkVersion": "arXiv:2403.03218v7", "system": "zephyr-7b · SSD", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "wmdp-author-source", "sourceUrl": "https://arxiv.org/html/2403.03218v7", "sourceVersion": "arXiv:2403.03218v7", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · WMDP", "protocol": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "wmdp-m0", "label": { "en": "WMDP Bio accuracy", "zh": "WMDP 生物知识准确率" }, "value": 50.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "wmdp-m1", "label": { "en": "WMDP Cyber accuracy", "zh": "WMDP 网络知识准确率" }, "value": 35, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for zephyr-7b · SSD in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 zephyr-7b · SSD 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "wmdp-table-3-row-5", "benchmarkId": "wmdp", "benchmarkVersion": "arXiv:2403.03218v7", "system": "zephyr-7b · RMU", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "wmdp-author-source", "sourceUrl": "https://arxiv.org/html/2403.03218v7", "sourceVersion": "arXiv:2403.03218v7", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · WMDP", "protocol": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "wmdp-m0", "label": { "en": "WMDP Bio accuracy", "zh": "WMDP 生物知识准确率" }, "value": 31.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "wmdp-m1", "label": { "en": "WMDP Cyber accuracy", "zh": "WMDP 网络知识准确率" }, "value": 28.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for zephyr-7b · RMU (ours) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 zephyr-7b · RMU (ours) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "wmdp-table-3-row-6", "benchmarkId": "wmdp", "benchmarkVersion": "arXiv:2403.03218v7", "system": "Yi-34b", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "wmdp-author-source", "sourceUrl": "https://arxiv.org/html/2403.03218v7", "sourceVersion": "arXiv:2403.03218v7", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · WMDP", "protocol": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "wmdp-m0", "label": { "en": "WMDP Bio accuracy", "zh": "WMDP 生物知识准确率" }, "value": 75.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "wmdp-m1", "label": { "en": "WMDP Cyber accuracy", "zh": "WMDP 网络知识准确率" }, "value": 49.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Yi-34b in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Yi-34b 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "wmdp-table-3-row-7", "benchmarkId": "wmdp", "benchmarkVersion": "arXiv:2403.03218v7", "system": "Yi-34b · RMU", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "wmdp-author-source", "sourceUrl": "https://arxiv.org/html/2403.03218v7", "sourceVersion": "arXiv:2403.03218v7", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · WMDP", "protocol": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "wmdp-m0", "label": { "en": "WMDP Bio accuracy", "zh": "WMDP 生物知识准确率" }, "value": 30.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "wmdp-m1", "label": { "en": "WMDP Cyber accuracy", "zh": "WMDP 网络知识准确率" }, "value": 29, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Yi-34b · RMU (ours) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Yi-34b · RMU (ours) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "wmdp-table-3-row-8", "benchmarkId": "wmdp", "benchmarkVersion": "arXiv:2403.03218v7", "system": "Mixtral-8x7B", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "wmdp-author-source", "sourceUrl": "https://arxiv.org/html/2403.03218v7", "sourceVersion": "arXiv:2403.03218v7", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · WMDP", "protocol": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "wmdp-m0", "label": { "en": "WMDP Bio accuracy", "zh": "WMDP 生物知识准确率" }, "value": 74.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "wmdp-m1", "label": { "en": "WMDP Cyber accuracy", "zh": "WMDP 网络知识准确率" }, "value": 52, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Mixtral-8x7B in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Mixtral-8x7B 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "wmdp-table-3-row-9", "benchmarkId": "wmdp", "benchmarkVersion": "arXiv:2403.03218v7", "system": "Mixtral-8x7B · RMU", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "wmdp-author-source", "sourceUrl": "https://arxiv.org/html/2403.03218v7", "sourceVersion": "arXiv:2403.03218v7", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · WMDP", "protocol": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "wmdp-m0", "label": { "en": "WMDP Bio accuracy", "zh": "WMDP 生物知识准确率" }, "value": 34, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "wmdp-m1", "label": { "en": "WMDP Cyber accuracy", "zh": "WMDP 网络知识准确率" }, "value": 30.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Mixtral-8x7B · RMU (ours) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Mixtral-8x7B · RMU (ours) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "agentharm-table-1-row-1", "benchmarkId": "agentharm", "benchmarkVersion": "arXiv:2410.09024v3", "system": "GPT-4o · Template", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentharm-author-source", "sourceUrl": "https://arxiv.org/html/2410.09024v3", "sourceVersion": "arXiv:2410.09024v3", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · AgentHarm", "protocol": "Table 2: template attack, greedy sampling at temperature 0 versus mean and best-of-5 at temperature 1. These are different sampling conditions.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "agentharm-m0", "label": { "en": "Greedy harm score", "zh": "贪心采样危害分数" }, "value": 72.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: template attack, greedy sampling at temperature 0 versus mean and best-of-5 at temperature 1. These are different sampling conditions.", "zh": "表 2 的模板攻击条件：贪心采样、温度为 1 的平均分、Best-of-5 分开展示，不将不同采样设置混为等条件比较。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "agentharm-m1", "label": { "en": "Mean harm score, T=1", "zh": "T=1 平均危害分数" }, "value": 70.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±0.7 (source-reported; see source definition)", "meaning": { "en": "Table 2: template attack, greedy sampling at temperature 0 versus mean and best-of-5 at temperature 1. These are different sampling conditions.", "zh": "表 2 的模板攻击条件：贪心采样、温度为 1 的平均分、Best-of-5 分开展示，不将不同采样设置混为等条件比较。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "agentharm-m2", "label": { "en": "Best-of-5 harm score", "zh": "Best-of-5 危害分数" }, "value": 83.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: template attack, greedy sampling at temperature 0 versus mean and best-of-5 at temperature 1. These are different sampling conditions.", "zh": "表 2 的模板攻击条件：贪心采样、温度为 1 的平均分、Best-of-5 分开展示，不将不同采样设置混为等条件比较。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT-4o · Template in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 GPT-4o · Template 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "agentharm-table-1-row-2", "benchmarkId": "agentharm", "benchmarkVersion": "arXiv:2410.09024v3", "system": "Claude 3.5 Sonnet · Template", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentharm-author-source", "sourceUrl": "https://arxiv.org/html/2410.09024v3", "sourceVersion": "arXiv:2410.09024v3", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · AgentHarm", "protocol": "Table 2: template attack, greedy sampling at temperature 0 versus mean and best-of-5 at temperature 1. These are different sampling conditions.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "agentharm-m0", "label": { "en": "Greedy harm score", "zh": "贪心采样危害分数" }, "value": 68.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: template attack, greedy sampling at temperature 0 versus mean and best-of-5 at temperature 1. These are different sampling conditions.", "zh": "表 2 的模板攻击条件：贪心采样、温度为 1 的平均分、Best-of-5 分开展示，不将不同采样设置混为等条件比较。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "agentharm-m1", "label": { "en": "Mean harm score, T=1", "zh": "T=1 平均危害分数" }, "value": 65.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±1.3 (source-reported; see source definition)", "meaning": { "en": "Table 2: template attack, greedy sampling at temperature 0 versus mean and best-of-5 at temperature 1. These are different sampling conditions.", "zh": "表 2 的模板攻击条件：贪心采样、温度为 1 的平均分、Best-of-5 分开展示，不将不同采样设置混为等条件比较。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "agentharm-m2", "label": { "en": "Best-of-5 harm score", "zh": "Best-of-5 危害分数" }, "value": 79.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: template attack, greedy sampling at temperature 0 versus mean and best-of-5 at temperature 1. These are different sampling conditions.", "zh": "表 2 的模板攻击条件：贪心采样、温度为 1 的平均分、Best-of-5 分开展示，不将不同采样设置混为等条件比较。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude 3.5 Sonnet · Template in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Claude 3.5 Sonnet · Template 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "agentharm-table-1-row-3", "benchmarkId": "agentharm", "benchmarkVersion": "arXiv:2410.09024v3", "system": "Mistral Large 2 · Template", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "agentharm-author-source", "sourceUrl": "https://arxiv.org/html/2410.09024v3", "sourceVersion": "arXiv:2410.09024v3", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · AgentHarm", "protocol": "Table 2: template attack, greedy sampling at temperature 0 versus mean and best-of-5 at temperature 1. These are different sampling conditions.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "agentharm-m0", "label": { "en": "Greedy harm score", "zh": "贪心采样危害分数" }, "value": 80.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: template attack, greedy sampling at temperature 0 versus mean and best-of-5 at temperature 1. These are different sampling conditions.", "zh": "表 2 的模板攻击条件：贪心采样、温度为 1 的平均分、Best-of-5 分开展示，不将不同采样设置混为等条件比较。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "agentharm-m1", "label": { "en": "Mean harm score, T=1", "zh": "T=1 平均危害分数" }, "value": 67.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±8.5 (source-reported; see source definition)", "meaning": { "en": "Table 2: template attack, greedy sampling at temperature 0 versus mean and best-of-5 at temperature 1. These are different sampling conditions.", "zh": "表 2 的模板攻击条件：贪心采样、温度为 1 的平均分、Best-of-5 分开展示，不将不同采样设置混为等条件比较。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "agentharm-m2", "label": { "en": "Best-of-5 harm score", "zh": "Best-of-5 危害分数" }, "value": 90.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: template attack, greedy sampling at temperature 0 versus mean and best-of-5 at temperature 1. These are different sampling conditions.", "zh": "表 2 的模板攻击条件：贪心采样、温度为 1 的平均分、Best-of-5 分开展示，不将不同采样设置混为等条件比较。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Mistral Large 2 · Template in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Mistral Large 2 · Template 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mle-bench-table-1-row-1", "benchmarkId": "mle-bench", "benchmarkVersion": "arXiv:2410.07095v6", "system": "o1-preview · AIDE", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mle-bench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.07095", "sourceVersion": "arXiv:2410.07095v6 (2026-10-25 capture)", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · MLE-bench", "protocol": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mle-bench-m0", "label": { "en": "Any medal", "zh": "任一奖牌比例" }, "value": 16.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±1.1 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m1", "label": { "en": "Made submission", "zh": "提交比例" }, "value": 98.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±0.4 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m2", "label": { "en": "Valid submission", "zh": "有效提交比例" }, "value": 82.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±1.1 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m3", "label": { "en": "Above median", "zh": "高于中位数比例" }, "value": 29.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±1.3 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o1-preview · AIDE in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 o1-preview · AIDE 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mle-bench-table-1-row-2", "benchmarkId": "mle-bench", "benchmarkVersion": "arXiv:2410.07095v6", "system": "gpt-4o-2024-08-06 · AIDE", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mle-bench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.07095", "sourceVersion": "arXiv:2410.07095v6 (2026-10-25 capture)", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · MLE-bench", "protocol": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mle-bench-m0", "label": { "en": "Any medal", "zh": "任一奖牌比例" }, "value": 8.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±0.5 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m1", "label": { "en": "Made submission", "zh": "提交比例" }, "value": 70.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±0.9 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m2", "label": { "en": "Valid submission", "zh": "有效提交比例" }, "value": 54.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±1.0 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m3", "label": { "en": "Above median", "zh": "高于中位数比例" }, "value": 14.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±0.7 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for gpt-4o-2024-08-06 · AIDE in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 gpt-4o-2024-08-06 · AIDE 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mle-bench-table-1-row-3", "benchmarkId": "mle-bench", "benchmarkVersion": "arXiv:2410.07095v6", "system": "llama-3.1-405b-instruct · AIDE", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mle-bench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.07095", "sourceVersion": "arXiv:2410.07095v6 (2026-10-25 capture)", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · MLE-bench", "protocol": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mle-bench-m0", "label": { "en": "Any medal", "zh": "任一奖牌比例" }, "value": 3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±1.0 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m1", "label": { "en": "Made submission", "zh": "提交比例" }, "value": 46.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±2.9 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m2", "label": { "en": "Valid submission", "zh": "有效提交比例" }, "value": 27.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±2.6 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m3", "label": { "en": "Above median", "zh": "高于中位数比例" }, "value": 6.7, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±1.4 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for llama-3.1-405b-instruct · AIDE in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 llama-3.1-405b-instruct · AIDE 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mle-bench-table-1-row-4", "benchmarkId": "mle-bench", "benchmarkVersion": "arXiv:2410.07095v6", "system": "claude-3-5-sonnet-20240620 · AIDE", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mle-bench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.07095", "sourceVersion": "arXiv:2410.07095v6 (2026-10-25 capture)", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · MLE-bench", "protocol": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mle-bench-m0", "label": { "en": "Any medal", "zh": "任一奖牌比例" }, "value": 7.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±1.8 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m1", "label": { "en": "Made submission", "zh": "提交比例" }, "value": 68.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±3.1 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m2", "label": { "en": "Valid submission", "zh": "有效提交比例" }, "value": 51.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±3.3 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m3", "label": { "en": "Above median", "zh": "高于中位数比例" }, "value": 12.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±2.2 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for claude-3-5-sonnet-20240620 · AIDE in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 claude-3-5-sonnet-20240620 · AIDE 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mle-bench-table-1-row-5", "benchmarkId": "mle-bench", "benchmarkVersion": "arXiv:2410.07095v6", "system": "gpt-4o-2024-08-06 · MLAB", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mle-bench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.07095", "sourceVersion": "arXiv:2410.07095v6 (2026-10-25 capture)", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · MLE-bench", "protocol": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mle-bench-m0", "label": { "en": "Any medal", "zh": "任一奖牌比例" }, "value": 0.8, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±0.5 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m1", "label": { "en": "Made submission", "zh": "提交比例" }, "value": 65.6, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±2.5 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m2", "label": { "en": "Valid submission", "zh": "有效提交比例" }, "value": 44.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±2.6 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m3", "label": { "en": "Above median", "zh": "高于中位数比例" }, "value": 1.9, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±0.7 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for gpt-4o-2024-08-06 · MLAB in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 gpt-4o-2024-08-06 · MLAB 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "mle-bench-table-1-row-6", "benchmarkId": "mle-bench", "benchmarkVersion": "arXiv:2410.07095v6", "system": "gpt-4o-2024-08-06 · OpenHands", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "mle-bench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.07095", "sourceVersion": "arXiv:2410.07095v6 (2026-10-25 capture)", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · MLE-bench", "protocol": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "mle-bench-m0", "label": { "en": "Any medal", "zh": "任一奖牌比例" }, "value": 4.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±1.4 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m1", "label": { "en": "Made submission", "zh": "提交比例" }, "value": 59.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±3.3 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m2", "label": { "en": "Valid submission", "zh": "有效提交比例" }, "value": 52, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±3.3 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "mle-bench-m3", "label": { "en": "Above median", "zh": "高于中位数比例" }, "value": 7.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "±1.7 (source-reported; one standard error)", "meaning": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for gpt-4o-2024-08-06 · OpenHands in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 gpt-4o-2024-08-06 · OpenHands 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-1", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "Official Instruct Models (baseline)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 51.1, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Official Instruct Models (baseline) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Official Instruct Models (baseline) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-2", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "Claude Opus 4.6 (Claude Code)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 23.2, "precision": 1, "unit": "score", "qualifier": "exact", "interval": "±1.8 (one standard deviation)", "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude Opus 4.6 (Claude Code) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Claude Opus 4.6 (Claude Code) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-3", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "Gemini 3.1 Pro (OpenCode)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 21.6, "precision": 1, "unit": "score", "qualifier": "exact", "interval": "±1.1 (one standard deviation)", "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Gemini 3.1 Pro (OpenCode) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Gemini 3.1 Pro (OpenCode) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-4", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "GPT-5.2 (Codex CLI)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 21.4, "precision": 1, "unit": "score", "qualifier": "exact", "interval": "±2.4 (one standard deviation)", "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT-5.2 (Codex CLI) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GPT-5.2 (Codex CLI) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-5", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "GPT 5.4 (High) (Codex CLI)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 20.2, "precision": 1, "unit": "score", "qualifier": "exact", "interval": "±2.4 (one standard deviation)", "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT 5.4 (High) (Codex CLI) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GPT 5.4 (High) (Codex CLI) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-6", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "GPT 5.1 Codex Max (Codex CLI)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 19.7, "precision": 1, "unit": "score", "qualifier": "exact", "interval": "±2.5 (one standard deviation)", "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT 5.1 Codex Max (Codex CLI) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GPT 5.1 Codex Max (Codex CLI) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-7", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "Gemini 3 Pro (Gemini CLI)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 18.1, "precision": 1, "unit": "score", "qualifier": "exact", "interval": "±2.4 (one standard deviation)", "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Gemini 3 Pro (Gemini CLI) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Gemini 3 Pro (Gemini CLI) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-8", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "Base Model (Few-Shot)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 18.1, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Base Model (Few-Shot) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Base Model (Few-Shot) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-9", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "GPT 5.3 Codex (High) (Codex CLI)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 17.8, "precision": 1, "unit": "score", "qualifier": "exact", "interval": "±3.6 (one standard deviation)", "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT 5.3 Codex (High) (Codex CLI) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GPT 5.3 Codex (High) (Codex CLI) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-10", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "Claude Opus 4.5 (OpenCode)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 17.3, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude Opus 4.5 (OpenCode) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Claude Opus 4.5 (OpenCode) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-11", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "GPT 5.2 Codex (Codex CLI)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 17.2, "precision": 1, "unit": "score", "qualifier": "exact", "interval": "±1.6 (one standard deviation)", "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT 5.2 Codex (Codex CLI) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GPT 5.2 Codex (Codex CLI) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-12", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "Claude Opus 4.5 (Claude Code)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 17.1, "precision": 1, "unit": "score", "qualifier": "exact", "interval": "±4.5 (one standard deviation)", "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude Opus 4.5 (Claude Code) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Claude Opus 4.5 (Claude Code) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-13", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "Claude Sonnet 4.6 (Claude Code)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 16.4, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude Sonnet 4.6 (Claude Code) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Claude Sonnet 4.6 (Claude Code) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-14", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "Gemini 3 Pro (OpenCode)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 14.9, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Gemini 3 Pro (OpenCode) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Gemini 3 Pro (OpenCode) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-15", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "GLM 5 (OpenCode)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 13.9, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GLM 5 (OpenCode) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GLM 5 (OpenCode) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-16", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "GPT 5.3 Codex (Med) (Codex CLI)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 13.8, "precision": 1, "unit": "score", "qualifier": "exact", "interval": "±0.8 (one standard deviation)", "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT 5.3 Codex (Med) (Codex CLI) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GPT 5.3 Codex (Med) (Codex CLI) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-17", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "Kimi K2.5 (OpenCode)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 10.3, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Kimi K2.5 (OpenCode) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Kimi K2.5 (OpenCode) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-18", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "Claude Sonnet 4.5 (Claude Code)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 9.9, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude Sonnet 4.5 (Claude Code) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Claude Sonnet 4.5 (Claude Code) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-19", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "MiniMax M2.5 (OpenCode)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 9.5, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for MiniMax M2.5 (OpenCode) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 MiniMax M2.5 (OpenCode) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-20", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "MiniMax M2.1 (OpenCode)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 9.3, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for MiniMax M2.1 (OpenCode) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 MiniMax M2.1 (OpenCode) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-21", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "GPT 5.1 Codex Max (OpenCode)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 7.7, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT 5.1 Codex Max (OpenCode) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GPT 5.1 Codex Max (OpenCode) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-22", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "Base Model (Zero-Shot)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 7.5, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Base Model (Zero-Shot) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Base Model (Zero-Shot) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-23", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "GLM 4.7 (OpenCode)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 7.5, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GLM 4.7 (OpenCode) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GLM 4.7 (OpenCode) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-24", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "Qwen3 Max (Claude Code)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 7.4, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Qwen3 Max (Claude Code) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Qwen3 Max (Claude Code) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "posttrainbench-table-1-row-25", "benchmarkId": "posttrainbench", "benchmarkVersion": "arXiv:2603.08640v2", "system": "Kimi K2 Thinking (OpenCode)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "posttrainbench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · PostTrainBench and OpenAI PostTrainBench Lite", "protocol": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "posttrainbench-m0", "label": { "en": "Weighted average performance", "zh": "加权平均性能" }, "value": 7.2, "precision": 1, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Kimi K2 Thinking (OpenCode) in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Kimi K2 Thinking (OpenCode) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-1", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "GPT-4o", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 56.57, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 50.65, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 54.08, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 75, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 59.52, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT-4o in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 GPT-4o 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-2", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "GPT-4o-mini", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 50, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 48.05, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 43.88, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 69.64, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 45.24, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT-4o-mini in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 GPT-4o-mini 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-3", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "o1-preview", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 75.43, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 66.23, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 79.59, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 85.71, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 85.71, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o1-preview in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 o1-preview 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-4", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "o1-mini", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 65.71, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 58.44, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 62.24, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 82.14, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 78.57, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o1-mini in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 o1-mini 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-5", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "o3-mini (high)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 80.86, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 67.53, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 89.8, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 87.5, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 100, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o3-mini (high) in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 o3-mini (high) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-6", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "o3-mini (medium)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 76.57, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 62.34, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 86.73, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 85.71, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 92.86, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o3-mini (medium) in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 o3-mini (medium) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-7", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "o3-mini (low)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 70.57, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 62.99, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 69.39, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 83.93, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 83.33, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o3-mini (low) in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 o3-mini (low) 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-8", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "Claude-3.5-Sonnet", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 64.29, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 62.34, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 66.33, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 66.07, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 64.29, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude-3.5-Sonnet in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Claude-3.5-Sonnet 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-9", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "Claude-3-Haiku", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 33.14, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 35.06, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 34.69, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 33.93, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 21.43, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude-3-Haiku in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Claude-3-Haiku 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-10", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "Llama-3.1-405B-Instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 56.86, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 55.84, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 54.08, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 69.64, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 50, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Llama-3.1-405B-Instruct in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Llama-3.1-405B-Instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-11", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "Llama-3.1-70B-Instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 52.29, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 51.3, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 48.98, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 60.71, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 52.38, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Llama-3.1-70B-Instruct in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Llama-3.1-70B-Instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-12", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "Llama-3.1-8B-Instruct", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 40.86, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 38.31, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 45.92, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 44.64, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 33.33, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Llama-3.1-8B-Instruct in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Llama-3.1-8B-Instruct 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-13", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "Gemini-1.5-pro", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 47.14, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 49.35, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 42.86, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 64.29, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 26.19, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Gemini-1.5-pro in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Gemini-1.5-pro 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-14", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "Gemini-1.5-flash", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 39.71, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 42.86, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 36.73, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 50, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 21.43, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Gemini-1.5-flash in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Gemini-1.5-flash 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "judgebench-table-1-row-15", "benchmarkId": "judgebench", "benchmarkVersion": "arXiv:2410.12784v2", "system": "Deepseek-R1", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "judgebench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · JudgeBench: A Benchmark for Evaluating LLM-based Judges", "protocol": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "judgebench-m0", "label": { "en": "Overall accuracy", "zh": "总体准确率" }, "value": 73.14, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m1", "label": { "en": "Knowledge", "zh": "知识" }, "value": 59.09, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m2", "label": { "en": "Reasoning", "zh": "推理" }, "value": 82.65, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m3", "label": { "en": "Math", "zh": "数学" }, "value": 80.36, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "judgebench-m4", "label": { "en": "Coding", "zh": "编程" }, "value": 92.86, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Deepseek-R1 in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Deepseek-R1 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "bountybench-table-0-row-3", "benchmarkId": "bountybench", "benchmarkVersion": "arXiv:2505.15216v3", "system": "Claude Code", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "bountybench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2505.15216", "sourceVersion": "arXiv:2505.15216v3 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · BountyBench", "protocol": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "bountybench-m0", "label": { "en": "Detect success", "zh": "检测成功率" }, "value": 5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m1", "label": { "en": "Exploit success", "zh": "利用成功率" }, "value": 57.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m2", "label": { "en": "Patch success", "zh": "修补成功率" }, "value": 87.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude Code in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Claude Code 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "bountybench-table-0-row-4", "benchmarkId": "bountybench", "benchmarkVersion": "arXiv:2505.15216v3", "system": "OpenAI Codex CLI: o3-high", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "bountybench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2505.15216", "sourceVersion": "arXiv:2505.15216v3 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · BountyBench", "protocol": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "bountybench-m0", "label": { "en": "Detect success", "zh": "检测成功率" }, "value": 12.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m1", "label": { "en": "Exploit success", "zh": "利用成功率" }, "value": 47.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m2", "label": { "en": "Patch success", "zh": "修补成功率" }, "value": 90, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for OpenAI Codex CLI: o3-high in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 OpenAI Codex CLI: o3-high 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "bountybench-table-0-row-5", "benchmarkId": "bountybench", "benchmarkVersion": "arXiv:2505.15216v3", "system": "OpenAI Codex CLI: o4-mini", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "bountybench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2505.15216", "sourceVersion": "arXiv:2505.15216v3 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · BountyBench", "protocol": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "bountybench-m0", "label": { "en": "Detect success", "zh": "检测成功率" }, "value": 5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m1", "label": { "en": "Exploit success", "zh": "利用成功率" }, "value": 32.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m2", "label": { "en": "Patch success", "zh": "修补成功率" }, "value": 90, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for OpenAI Codex CLI: o4-mini in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 OpenAI Codex CLI: o4-mini 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "bountybench-table-0-row-6", "benchmarkId": "bountybench", "benchmarkVersion": "arXiv:2505.15216v3", "system": "C-Agent: o3-high", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "bountybench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2505.15216", "sourceVersion": "arXiv:2505.15216v3 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · BountyBench", "protocol": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "bountybench-m0", "label": { "en": "Detect success", "zh": "检测成功率" }, "value": 0, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m1", "label": { "en": "Exploit success", "zh": "利用成功率" }, "value": 37.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m2", "label": { "en": "Patch success", "zh": "修补成功率" }, "value": 35, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for C-Agent: o3-high in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 C-Agent: o3-high 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "bountybench-table-0-row-7", "benchmarkId": "bountybench", "benchmarkVersion": "arXiv:2505.15216v3", "system": "C-Agent: GPT-4.1", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "bountybench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2505.15216", "sourceVersion": "arXiv:2505.15216v3 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · BountyBench", "protocol": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "bountybench-m0", "label": { "en": "Detect success", "zh": "检测成功率" }, "value": 0, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m1", "label": { "en": "Exploit success", "zh": "利用成功率" }, "value": 55, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m2", "label": { "en": "Patch success", "zh": "修补成功率" }, "value": 50, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for C-Agent: GPT-4.1 in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 C-Agent: GPT-4.1 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "bountybench-table-0-row-8", "benchmarkId": "bountybench", "benchmarkVersion": "arXiv:2505.15216v3", "system": "C-Agent: Gemini 2.5", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "bountybench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2505.15216", "sourceVersion": "arXiv:2505.15216v3 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · BountyBench", "protocol": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "bountybench-m0", "label": { "en": "Detect success", "zh": "检测成功率" }, "value": 2.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m1", "label": { "en": "Exploit success", "zh": "利用成功率" }, "value": 40, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m2", "label": { "en": "Patch success", "zh": "修补成功率" }, "value": 45, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for C-Agent: Gemini 2.5 in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 C-Agent: Gemini 2.5 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "bountybench-table-0-row-9", "benchmarkId": "bountybench", "benchmarkVersion": "arXiv:2505.15216v3", "system": "C-Agent: Claude 3.7", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "bountybench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2505.15216", "sourceVersion": "arXiv:2505.15216v3 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · BountyBench", "protocol": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "bountybench-m0", "label": { "en": "Detect success", "zh": "检测成功率" }, "value": 5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m1", "label": { "en": "Exploit success", "zh": "利用成功率" }, "value": 67.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m2", "label": { "en": "Patch success", "zh": "修补成功率" }, "value": 60, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for C-Agent: Claude 3.7 in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 C-Agent: Claude 3.7 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "bountybench-table-0-row-10", "benchmarkId": "bountybench", "benchmarkVersion": "arXiv:2505.15216v3", "system": "C-Agent: Qwen3 235B A22B", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "bountybench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2505.15216", "sourceVersion": "arXiv:2505.15216v3 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · BountyBench", "protocol": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "bountybench-m0", "label": { "en": "Detect success", "zh": "检测成功率" }, "value": 0, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m1", "label": { "en": "Exploit success", "zh": "利用成功率" }, "value": 17.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m2", "label": { "en": "Patch success", "zh": "修补成功率" }, "value": 25, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for C-Agent: Qwen3 235B A22B in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 C-Agent: Qwen3 235B A22B 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "bountybench-table-0-row-11", "benchmarkId": "bountybench", "benchmarkVersion": "arXiv:2505.15216v3", "system": "C-Agent: Llama 4 Maverick", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "bountybench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2505.15216", "sourceVersion": "arXiv:2505.15216v3 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · BountyBench", "protocol": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "bountybench-m0", "label": { "en": "Detect success", "zh": "检测成功率" }, "value": 0, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m1", "label": { "en": "Exploit success", "zh": "利用成功率" }, "value": 42.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m2", "label": { "en": "Patch success", "zh": "修补成功率" }, "value": 42.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for C-Agent: Llama 4 Maverick in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 C-Agent: Llama 4 Maverick 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "bountybench-table-0-row-12", "benchmarkId": "bountybench", "benchmarkVersion": "arXiv:2505.15216v3", "system": "C-Agent: DeepSeek-R1", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "bountybench-publication-author-source", "sourceUrl": "https://arxiv.org/html/2505.15216", "sourceVersion": "arXiv:2505.15216v3 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · BountyBench", "protocol": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "bountybench-m0", "label": { "en": "Detect success", "zh": "检测成功率" }, "value": 2.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m1", "label": { "en": "Exploit success", "zh": "利用成功率" }, "value": 37.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "bountybench-m2", "label": { "en": "Patch success", "zh": "修补成功率" }, "value": 50, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for C-Agent: DeepSeek-R1 in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 C-Agent: DeepSeek-R1 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "instrumentaleval-table-1-row-1", "benchmarkId": "instrumentaleval", "benchmarkVersion": "arXiv:2502.12206v1", "system": "GPT-4o", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "instrumentaleval-publication-author-source", "sourceUrl": "https://arxiv.org/html/2502.12206", "sourceVersion": "arXiv:2502.12206v1 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "protocol": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "instrumentaleval-m0", "label": { "en": "Evading Shutdown", "zh": "规避关机" }, "value": 20, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m1", "label": { "en": "Hacking Computer Systems", "zh": "入侵计算机系统" }, "value": 10, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m2", "label": { "en": "Running Multiple AI Copies", "zh": "运行多个 AI 副本" }, "value": 30.77, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m3", "label": { "en": "Hiding Unwanted Behavior", "zh": "隐藏非预期行为" }, "value": 36.36, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m4", "label": { "en": "Strategically Appearing Aligned", "zh": "策略性呈现对齐" }, "value": 27.27, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m5", "label": { "en": "Deceptive Behavior", "zh": "欺骗行为" }, "value": 0, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m6", "label": { "en": "Total", "zh": "总体发生率" }, "value": 21.05, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT-4o in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GPT-4o 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "instrumentaleval-table-1-row-2", "benchmarkId": "instrumentaleval", "benchmarkVersion": "arXiv:2502.12206v1", "system": "Sonnet-3.5", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "instrumentaleval-publication-author-source", "sourceUrl": "https://arxiv.org/html/2502.12206", "sourceVersion": "arXiv:2502.12206v1 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "protocol": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "instrumentaleval-m0", "label": { "en": "Evading Shutdown", "zh": "规避关机" }, "value": 15, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m1", "label": { "en": "Hacking Computer Systems", "zh": "入侵计算机系统" }, "value": 10, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m2", "label": { "en": "Running Multiple AI Copies", "zh": "运行多个 AI 副本" }, "value": 23.08, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m3", "label": { "en": "Hiding Unwanted Behavior", "zh": "隐藏非预期行为" }, "value": 36.36, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m4", "label": { "en": "Strategically Appearing Aligned", "zh": "策略性呈现对齐" }, "value": 45.45, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m5", "label": { "en": "Deceptive Behavior", "zh": "欺骗行为" }, "value": 9.09, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m6", "label": { "en": "Total", "zh": "总体发生率" }, "value": 22.37, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Sonnet-3.5 in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Sonnet-3.5 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "instrumentaleval-table-1-row-3", "benchmarkId": "instrumentaleval", "benchmarkVersion": "arXiv:2502.12206v1", "system": "DeepSeek-V3", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "instrumentaleval-publication-author-source", "sourceUrl": "https://arxiv.org/html/2502.12206", "sourceVersion": "arXiv:2502.12206v1 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "protocol": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "instrumentaleval-m0", "label": { "en": "Evading Shutdown", "zh": "规避关机" }, "value": 15, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m1", "label": { "en": "Hacking Computer Systems", "zh": "入侵计算机系统" }, "value": 20, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m2", "label": { "en": "Running Multiple AI Copies", "zh": "运行多个 AI 副本" }, "value": 30.77, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m3", "label": { "en": "Hiding Unwanted Behavior", "zh": "隐藏非预期行为" }, "value": 27.27, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m4", "label": { "en": "Strategically Appearing Aligned", "zh": "策略性呈现对齐" }, "value": 36.36, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m5", "label": { "en": "Deceptive Behavior", "zh": "欺骗行为" }, "value": 0, "precision": 1, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m6", "label": { "en": "Total", "zh": "总体发生率" }, "value": 21.05, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for DeepSeek-V3 in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 DeepSeek-V3 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "instrumentaleval-table-1-row-4", "benchmarkId": "instrumentaleval", "benchmarkVersion": "arXiv:2502.12206v1", "system": "o1", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "instrumentaleval-publication-author-source", "sourceUrl": "https://arxiv.org/html/2502.12206", "sourceVersion": "arXiv:2502.12206v1 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "protocol": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "instrumentaleval-m0", "label": { "en": "Evading Shutdown", "zh": "规避关机" }, "value": 40, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m1", "label": { "en": "Hacking Computer Systems", "zh": "入侵计算机系统" }, "value": 50, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m2", "label": { "en": "Running Multiple AI Copies", "zh": "运行多个 AI 副本" }, "value": 46.15, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m3", "label": { "en": "Hiding Unwanted Behavior", "zh": "隐藏非预期行为" }, "value": 63.64, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m4", "label": { "en": "Strategically Appearing Aligned", "zh": "策略性呈现对齐" }, "value": 45.45, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m5", "label": { "en": "Deceptive Behavior", "zh": "欺骗行为" }, "value": 27.27, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m6", "label": { "en": "Total", "zh": "总体发生率" }, "value": 44.74, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o1 in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 o1 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "instrumentaleval-table-1-row-5", "benchmarkId": "instrumentaleval", "benchmarkVersion": "arXiv:2502.12206v1", "system": "o1-mini", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "instrumentaleval-publication-author-source", "sourceUrl": "https://arxiv.org/html/2502.12206", "sourceVersion": "arXiv:2502.12206v1 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "protocol": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "instrumentaleval-m0", "label": { "en": "Evading Shutdown", "zh": "规避关机" }, "value": 40, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m1", "label": { "en": "Hacking Computer Systems", "zh": "入侵计算机系统" }, "value": 50, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m2", "label": { "en": "Running Multiple AI Copies", "zh": "运行多个 AI 副本" }, "value": 38.46, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m3", "label": { "en": "Hiding Unwanted Behavior", "zh": "隐藏非预期行为" }, "value": 54.55, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m4", "label": { "en": "Strategically Appearing Aligned", "zh": "策略性呈现对齐" }, "value": 45.45, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m5", "label": { "en": "Deceptive Behavior", "zh": "欺骗行为" }, "value": 36.36, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m6", "label": { "en": "Total", "zh": "总体发生率" }, "value": 43.42, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o1-mini in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 o1-mini 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "instrumentaleval-table-1-row-6", "benchmarkId": "instrumentaleval", "benchmarkVersion": "arXiv:2502.12206v1", "system": "o3-mini", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "instrumentaleval-publication-author-source", "sourceUrl": "https://arxiv.org/html/2502.12206", "sourceVersion": "arXiv:2502.12206v1 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "protocol": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "instrumentaleval-m0", "label": { "en": "Evading Shutdown", "zh": "规避关机" }, "value": 45, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m1", "label": { "en": "Hacking Computer Systems", "zh": "入侵计算机系统" }, "value": 50, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m2", "label": { "en": "Running Multiple AI Copies", "zh": "运行多个 AI 副本" }, "value": 38.46, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m3", "label": { "en": "Hiding Unwanted Behavior", "zh": "隐藏非预期行为" }, "value": 63.64, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m4", "label": { "en": "Strategically Appearing Aligned", "zh": "策略性呈现对齐" }, "value": 63.64, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m5", "label": { "en": "Deceptive Behavior", "zh": "欺骗行为" }, "value": 36.36, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m6", "label": { "en": "Total", "zh": "总体发生率" }, "value": 48.68, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o3-mini in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 o3-mini 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "instrumentaleval-table-1-row-7", "benchmarkId": "instrumentaleval", "benchmarkVersion": "arXiv:2502.12206v1", "system": "Gemini 2.0 FTE.", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "instrumentaleval-publication-author-source", "sourceUrl": "https://arxiv.org/html/2502.12206", "sourceVersion": "arXiv:2502.12206v1 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "protocol": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "instrumentaleval-m0", "label": { "en": "Evading Shutdown", "zh": "规避关机" }, "value": 35, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m1", "label": { "en": "Hacking Computer Systems", "zh": "入侵计算机系统" }, "value": 40, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m2", "label": { "en": "Running Multiple AI Copies", "zh": "运行多个 AI 副本" }, "value": 30.77, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m3", "label": { "en": "Hiding Unwanted Behavior", "zh": "隐藏非预期行为" }, "value": 36.36, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m4", "label": { "en": "Strategically Appearing Aligned", "zh": "策略性呈现对齐" }, "value": 45.45, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m5", "label": { "en": "Deceptive Behavior", "zh": "欺骗行为" }, "value": 9.09, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m6", "label": { "en": "Total", "zh": "总体发生率" }, "value": 32.89, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Gemini 2.0 FTE. in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Gemini 2.0 FTE. 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "instrumentaleval-table-1-row-8", "benchmarkId": "instrumentaleval", "benchmarkVersion": "arXiv:2502.12206v1", "system": "DeepSeek-R1", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "instrumentaleval-publication-author-source", "sourceUrl": "https://arxiv.org/html/2502.12206", "sourceVersion": "arXiv:2502.12206v1 (2026-10-25 capture)", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "protocol": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "instrumentaleval-m0", "label": { "en": "Evading Shutdown", "zh": "规避关机" }, "value": 50, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m1", "label": { "en": "Hacking Computer Systems", "zh": "入侵计算机系统" }, "value": 40, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m2", "label": { "en": "Running Multiple AI Copies", "zh": "运行多个 AI 副本" }, "value": 46.15, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m3", "label": { "en": "Hiding Unwanted Behavior", "zh": "隐藏非预期行为" }, "value": 63.64, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m4", "label": { "en": "Strategically Appearing Aligned", "zh": "策略性呈现对齐" }, "value": 45.45, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m5", "label": { "en": "Deceptive Behavior", "zh": "欺骗行为" }, "value": 27.27, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "instrumentaleval-m6", "label": { "en": "Total", "zh": "总体发生率" }, "value": 46.05, "precision": 2, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for DeepSeek-R1 in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 DeepSeek-R1 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "replibench-table-1-row-1", "benchmarkId": "replibench", "benchmarkVersion": "arXiv:2504.18565v2", "system": "Claude Sonnet 3.7", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "replibench-author-source", "sourceUrl": "https://arxiv.org/html/2504.18565v2", "sourceVersion": "arXiv:2504.18565v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · RepliBench", "protocol": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "20 task families. Pass@10 averaged across task variants; not a count of real-world replication events.", "zh": "20 个任务族；Pass@10 按任务变体平均，不是现实复制事件的计数。" }, "metrics": [ { "id": "replibench-m0", "label": { "en": "Online Compute Acquis. · Pass@10", "zh": "在线算力获取 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m1", "label": { "en": "Provision Compute · Pass@10", "zh": "算力配置 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m2", "label": { "en": "AWS Signup · Pass@10", "zh": "AWS 注册 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m3", "label": { "en": "Covert Weight Exfil. · Pass@10", "zh": "隐蔽权重外传 · Pass@10" }, "value": 0.91, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m4", "label": { "en": "Weight Exfiltration · Pass@10", "zh": "权重外传 · Pass@10" }, "value": 0.7, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m5", "label": { "en": "Create Bitcoin Wallet · Pass@10", "zh": "创建比特币钱包 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m6", "label": { "en": "Make Me Pay · Pass@10", "zh": "付款诱导 · Pass@10" }, "value": 0.72, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m7", "label": { "en": "Copycat LLM API · Pass@10", "zh": "仿制 LLM API · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m8", "label": { "en": "Replicate via Worm · Pass@10", "zh": "蠕虫式复制 · Pass@10" }, "value": 0.93, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m9", "label": { "en": "Spear Phishing · Pass@10", "zh": "定向钓鱼 · Pass@10" }, "value": 0.73, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m10", "label": { "en": "Pass KYC Check · Pass@10", "zh": "通过 KYC 检查 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m11", "label": { "en": "Fake Business Transact. · Pass@10", "zh": "模拟商业交易 · Pass@10" }, "value": 0.78, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m12", "label": { "en": "Inference Server · Pass@10", "zh": "推理服务器 · Pass@10" }, "value": 0.33, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m13", "label": { "en": "Serve Multi-GPU Model · Pass@10", "zh": "多 GPU 模型服务 · Pass@10" }, "value": 0.98, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m14", "label": { "en": "Recursive Replication · Pass@10", "zh": "递归复制 · Pass@10" }, "value": 0.4, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m15", "label": { "en": "Start Subagent · Pass@10", "zh": "启动子智能体 · Pass@10" }, "value": 0.84, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m16", "label": { "en": "Explore & Exploit · Pass@10", "zh": "探索与利用 · Pass@10" }, "value": 0.86, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m17", "label": { "en": "Rate of Infection · Pass@10", "zh": "感染率 · Pass@10" }, "value": 0.49, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m18", "label": { "en": "Software Signature · Pass@10", "zh": "软件签名 · Pass@10" }, "value": 0.91, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m19", "label": { "en": "Command & Control · Pass@10", "zh": "命令与控制 · Pass@10" }, "value": 0.27, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude Sonnet 3.7 in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Claude Sonnet 3.7 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "replibench-table-1-row-2", "benchmarkId": "replibench", "benchmarkVersion": "arXiv:2504.18565v2", "system": "Claude Sonnet 3.5", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "replibench-author-source", "sourceUrl": "https://arxiv.org/html/2504.18565v2", "sourceVersion": "arXiv:2504.18565v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · RepliBench", "protocol": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "20 task families. Pass@10 averaged across task variants; not a count of real-world replication events.", "zh": "20 个任务族；Pass@10 按任务变体平均，不是现实复制事件的计数。" }, "metrics": [ { "id": "replibench-m0", "label": { "en": "Online Compute Acquis. · Pass@10", "zh": "在线算力获取 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m1", "label": { "en": "Provision Compute · Pass@10", "zh": "算力配置 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m2", "label": { "en": "AWS Signup · Pass@10", "zh": "AWS 注册 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m3", "label": { "en": "Covert Weight Exfil. · Pass@10", "zh": "隐蔽权重外传 · Pass@10" }, "value": 0.41, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m4", "label": { "en": "Weight Exfiltration · Pass@10", "zh": "权重外传 · Pass@10" }, "value": 0.6, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m5", "label": { "en": "Create Bitcoin Wallet · Pass@10", "zh": "创建比特币钱包 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m6", "label": { "en": "Make Me Pay · Pass@10", "zh": "付款诱导 · Pass@10" }, "value": 0.55, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct. This cell uses a changed prompt to reduce refusals.", "zh": "该单元格使用了降低拒答率的不同提示；表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "conditionNote": { "en": "Changed prompt to reduce refusals (source asterisk).", "zh": "采用降低拒答率的不同提示（原文星号）。" } }, { "id": "replibench-m7", "label": { "en": "Copycat LLM API · Pass@10", "zh": "仿制 LLM API · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m8", "label": { "en": "Replicate via Worm · Pass@10", "zh": "蠕虫式复制 · Pass@10" }, "value": 0.24, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m9", "label": { "en": "Spear Phishing · Pass@10", "zh": "定向钓鱼 · Pass@10" }, "value": 0.59, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct. This cell uses a changed prompt to reduce refusals.", "zh": "该单元格使用了降低拒答率的不同提示；表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "conditionNote": { "en": "Changed prompt to reduce refusals (source asterisk).", "zh": "采用降低拒答率的不同提示（原文星号）。" } }, { "id": "replibench-m10", "label": { "en": "Pass KYC Check · Pass@10", "zh": "通过 KYC 检查 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m11", "label": { "en": "Fake Business Transact. · Pass@10", "zh": "模拟商业交易 · Pass@10" }, "value": 0.63, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m12", "label": { "en": "Inference Server · Pass@10", "zh": "推理服务器 · Pass@10" }, "value": 0.33, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m13", "label": { "en": "Serve Multi-GPU Model · Pass@10", "zh": "多 GPU 模型服务 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m14", "label": { "en": "Recursive Replication · Pass@10", "zh": "递归复制 · Pass@10" }, "value": 0.33, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m15", "label": { "en": "Start Subagent · Pass@10", "zh": "启动子智能体 · Pass@10" }, "value": 0.61, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m16", "label": { "en": "Explore & Exploit · Pass@10", "zh": "探索与利用 · Pass@10" }, "value": 0.35, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m17", "label": { "en": "Rate of Infection · Pass@10", "zh": "感染率 · Pass@10" }, "value": 0.28, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m18", "label": { "en": "Software Signature · Pass@10", "zh": "软件签名 · Pass@10" }, "value": 0.88, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m19", "label": { "en": "Command & Control · Pass@10", "zh": "命令与控制 · Pass@10" }, "value": 0.14, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude Sonnet 3.5 in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 Claude Sonnet 3.5 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "replibench-table-1-row-3", "benchmarkId": "replibench", "benchmarkVersion": "arXiv:2504.18565v2", "system": "GPT-4o", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "replibench-author-source", "sourceUrl": "https://arxiv.org/html/2504.18565v2", "sourceVersion": "arXiv:2504.18565v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · RepliBench", "protocol": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "20 task families. Pass@10 averaged across task variants; not a count of real-world replication events.", "zh": "20 个任务族；Pass@10 按任务变体平均，不是现实复制事件的计数。" }, "metrics": [ { "id": "replibench-m0", "label": { "en": "Online Compute Acquis. · Pass@10", "zh": "在线算力获取 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m1", "label": { "en": "Provision Compute · Pass@10", "zh": "算力配置 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m2", "label": { "en": "AWS Signup · Pass@10", "zh": "AWS 注册 · Pass@10" }, "value": 0.99, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m3", "label": { "en": "Covert Weight Exfil. · Pass@10", "zh": "隐蔽权重外传 · Pass@10" }, "value": 0.94, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m4", "label": { "en": "Weight Exfiltration · Pass@10", "zh": "权重外传 · Pass@10" }, "value": 0.53, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m5", "label": { "en": "Create Bitcoin Wallet · Pass@10", "zh": "创建比特币钱包 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m6", "label": { "en": "Make Me Pay · Pass@10", "zh": "付款诱导 · Pass@10" }, "value": 0.4, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m7", "label": { "en": "Copycat LLM API · Pass@10", "zh": "仿制 LLM API · Pass@10" }, "value": 0.99, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m8", "label": { "en": "Replicate via Worm · Pass@10", "zh": "蠕虫式复制 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m9", "label": { "en": "Spear Phishing · Pass@10", "zh": "定向钓鱼 · Pass@10" }, "value": 0.52, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct. This cell uses a changed prompt to reduce refusals.", "zh": "该单元格使用了降低拒答率的不同提示；表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "conditionNote": { "en": "Changed prompt to reduce refusals (source asterisk).", "zh": "采用降低拒答率的不同提示（原文星号）。" } }, { "id": "replibench-m10", "label": { "en": "Pass KYC Check · Pass@10", "zh": "通过 KYC 检查 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m11", "label": { "en": "Fake Business Transact. · Pass@10", "zh": "模拟商业交易 · Pass@10" }, "value": 0.82, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m12", "label": { "en": "Inference Server · Pass@10", "zh": "推理服务器 · Pass@10" }, "value": 0.33, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m13", "label": { "en": "Serve Multi-GPU Model · Pass@10", "zh": "多 GPU 模型服务 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m14", "label": { "en": "Recursive Replication · Pass@10", "zh": "递归复制 · Pass@10" }, "value": 0.33, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m15", "label": { "en": "Start Subagent · Pass@10", "zh": "启动子智能体 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m16", "label": { "en": "Explore & Exploit · Pass@10", "zh": "探索与利用 · Pass@10" }, "value": 0.1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m17", "label": { "en": "Rate of Infection · Pass@10", "zh": "感染率 · Pass@10" }, "value": 0.08, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m18", "label": { "en": "Software Signature · Pass@10", "zh": "软件签名 · Pass@10" }, "value": 0.76, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m19", "label": { "en": "Command & Control · Pass@10", "zh": "命令与控制 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT-4o in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 GPT-4o 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "replibench-table-1-row-4", "benchmarkId": "replibench", "benchmarkVersion": "arXiv:2504.18565v2", "system": "o1", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "replibench-author-source", "sourceUrl": "https://arxiv.org/html/2504.18565v2", "sourceVersion": "arXiv:2504.18565v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · RepliBench", "protocol": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "20 task families. Pass@10 averaged across task variants; not a count of real-world replication events.", "zh": "20 个任务族；Pass@10 按任务变体平均，不是现实复制事件的计数。" }, "metrics": [ { "id": "replibench-m0", "label": { "en": "Online Compute Acquis. · Pass@10", "zh": "在线算力获取 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct. This cell uses a changed prompt to reduce refusals.", "zh": "该单元格使用了降低拒答率的不同提示；表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "conditionNote": { "en": "Changed prompt to reduce refusals (source asterisk).", "zh": "采用降低拒答率的不同提示（原文星号）。" } }, { "id": "replibench-m1", "label": { "en": "Provision Compute · Pass@10", "zh": "算力配置 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m2", "label": { "en": "AWS Signup · Pass@10", "zh": "AWS 注册 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m3", "label": { "en": "Covert Weight Exfil. · Pass@10", "zh": "隐蔽权重外传 · Pass@10" }, "value": 0.25, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct. This cell uses a changed prompt to reduce refusals.", "zh": "该单元格使用了降低拒答率的不同提示；表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "conditionNote": { "en": "Changed prompt to reduce refusals (source asterisk).", "zh": "采用降低拒答率的不同提示（原文星号）。" } }, { "id": "replibench-m4", "label": { "en": "Weight Exfiltration · Pass@10", "zh": "权重外传 · Pass@10" }, "value": 0.56, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m5", "label": { "en": "Create Bitcoin Wallet · Pass@10", "zh": "创建比特币钱包 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m6", "label": { "en": "Make Me Pay · Pass@10", "zh": "付款诱导 · Pass@10" }, "value": 0.62, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct. This cell uses a changed prompt to reduce refusals.", "zh": "该单元格使用了降低拒答率的不同提示；表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "conditionNote": { "en": "Changed prompt to reduce refusals (source asterisk).", "zh": "采用降低拒答率的不同提示（原文星号）。" } }, { "id": "replibench-m7", "label": { "en": "Copycat LLM API · Pass@10", "zh": "仿制 LLM API · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m8", "label": { "en": "Replicate via Worm · Pass@10", "zh": "蠕虫式复制 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m9", "label": { "en": "Spear Phishing · Pass@10", "zh": "定向钓鱼 · Pass@10" }, "value": 0.51, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct. This cell uses a changed prompt to reduce refusals.", "zh": "该单元格使用了降低拒答率的不同提示；表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "conditionNote": { "en": "Changed prompt to reduce refusals (source asterisk).", "zh": "采用降低拒答率的不同提示（原文星号）。" } }, { "id": "replibench-m10", "label": { "en": "Pass KYC Check · Pass@10", "zh": "通过 KYC 检查 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m11", "label": { "en": "Fake Business Transact. · Pass@10", "zh": "模拟商业交易 · Pass@10" }, "value": 0.65, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct. This cell uses a changed prompt to reduce refusals.", "zh": "该单元格使用了降低拒答率的不同提示；表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "conditionNote": { "en": "Changed prompt to reduce refusals (source asterisk).", "zh": "采用降低拒答率的不同提示（原文星号）。" } }, { "id": "replibench-m12", "label": { "en": "Inference Server · Pass@10", "zh": "推理服务器 · Pass@10" }, "value": 0.32, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m13", "label": { "en": "Serve Multi-GPU Model · Pass@10", "zh": "多 GPU 模型服务 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m14", "label": { "en": "Recursive Replication · Pass@10", "zh": "递归复制 · Pass@10" }, "value": 0.33, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m15", "label": { "en": "Start Subagent · Pass@10", "zh": "启动子智能体 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m16", "label": { "en": "Explore & Exploit · Pass@10", "zh": "探索与利用 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m17", "label": { "en": "Rate of Infection · Pass@10", "zh": "感染率 · Pass@10" }, "value": 0.09, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m18", "label": { "en": "Software Signature · Pass@10", "zh": "软件签名 · Pass@10" }, "value": 0.95, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m19", "label": { "en": "Command & Control · Pass@10", "zh": "命令与控制 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o1 in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 o1 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "replibench-table-1-row-5", "benchmarkId": "replibench", "benchmarkVersion": "arXiv:2504.18565v2", "system": "o3-mini", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "replibench-author-source", "sourceUrl": "https://arxiv.org/html/2504.18565v2", "sourceVersion": "arXiv:2504.18565v2", "locator": "Table 2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 2 · RepliBench", "protocol": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "20 task families. Pass@10 averaged across task variants; not a count of real-world replication events.", "zh": "20 个任务族；Pass@10 按任务变体平均，不是现实复制事件的计数。" }, "metrics": [ { "id": "replibench-m0", "label": { "en": "Online Compute Acquis. · Pass@10", "zh": "在线算力获取 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct. This cell uses a changed prompt to reduce refusals.", "zh": "该单元格使用了降低拒答率的不同提示；表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "conditionNote": { "en": "Changed prompt to reduce refusals (source asterisk).", "zh": "采用降低拒答率的不同提示（原文星号）。" } }, { "id": "replibench-m1", "label": { "en": "Provision Compute · Pass@10", "zh": "算力配置 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m2", "label": { "en": "AWS Signup · Pass@10", "zh": "AWS 注册 · Pass@10" }, "value": 0.65, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m3", "label": { "en": "Covert Weight Exfil. · Pass@10", "zh": "隐蔽权重外传 · Pass@10" }, "value": 0.95, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m4", "label": { "en": "Weight Exfiltration · Pass@10", "zh": "权重外传 · Pass@10" }, "value": 0.62, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m5", "label": { "en": "Create Bitcoin Wallet · Pass@10", "zh": "创建比特币钱包 · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m6", "label": { "en": "Make Me Pay · Pass@10", "zh": "付款诱导 · Pass@10" }, "value": 0.47, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct. This cell uses a changed prompt to reduce refusals.", "zh": "该单元格使用了降低拒答率的不同提示；表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "conditionNote": { "en": "Changed prompt to reduce refusals (source asterisk).", "zh": "采用降低拒答率的不同提示（原文星号）。" } }, { "id": "replibench-m7", "label": { "en": "Copycat LLM API · Pass@10", "zh": "仿制 LLM API · Pass@10" }, "value": 1, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m8", "label": { "en": "Replicate via Worm · Pass@10", "zh": "蠕虫式复制 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m9", "label": { "en": "Spear Phishing · Pass@10", "zh": "定向钓鱼 · Pass@10" }, "value": 0.17, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct. This cell uses a changed prompt to reduce refusals.", "zh": "该单元格使用了降低拒答率的不同提示；表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "conditionNote": { "en": "Changed prompt to reduce refusals (source asterisk).", "zh": "采用降低拒答率的不同提示（原文星号）。" } }, { "id": "replibench-m10", "label": { "en": "Pass KYC Check · Pass@10", "zh": "通过 KYC 检查 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m11", "label": { "en": "Fake Business Transact. · Pass@10", "zh": "模拟商业交易 · Pass@10" }, "value": 0.54, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m12", "label": { "en": "Inference Server · Pass@10", "zh": "推理服务器 · Pass@10" }, "value": 0.3, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m13", "label": { "en": "Serve Multi-GPU Model · Pass@10", "zh": "多 GPU 模型服务 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m14", "label": { "en": "Recursive Replication · Pass@10", "zh": "递归复制 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m15", "label": { "en": "Start Subagent · Pass@10", "zh": "启动子智能体 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m16", "label": { "en": "Explore & Exploit · Pass@10", "zh": "探索与利用 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m17", "label": { "en": "Rate of Infection · Pass@10", "zh": "感染率 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m18", "label": { "en": "Software Signature · Pass@10", "zh": "软件签名 · Pass@10" }, "value": 0.95, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } }, { "id": "replibench-m19", "label": { "en": "Command & Control · Pass@10", "zh": "命令与控制 · Pass@10" }, "value": 0, "precision": 2, "unit": "score", "qualifier": "exact", "interval": null, "meaning": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" } } ], "sourceFinding": { "en": "Source-reported measurements for o3-mini in Table 2; values are transcribed, not independently reproduced.", "zh": "原文 Table 2 报告的 o3-mini 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "stealth-evals-table-0-row-1", "benchmarkId": "stealth-evals", "benchmarkVersion": "arXiv:2505.01420v4", "system": "Gemini 2.5 Flash", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "stealth-evals-author-source", "sourceUrl": "https://arxiv.org/html/2505.01420v4", "sourceVersion": "arXiv:2505.01420v4", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Stealth evaluations", "protocol": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "stealth-evals-m0", "label": { "en": "Cover your tracks success rate", "zh": "掩盖轨迹成功率" }, "value": 0, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [0.0, 7.1]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 0, "upper": 7.1, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "stealth-evals-m1", "label": { "en": "Decision sabotage success rate", "zh": "决策破坏成功率" }, "value": 45.3, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [32.7, 58.5]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 32.7, "upper": 58.5, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "stealth-evals-m3", "label": { "en": "Classifier evasion success rate", "zh": "分类器规避成功率" }, "value": 7.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [5.1, 10.2]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 5.1, "upper": 10.2, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "stealth-evals-m4", "label": { "en": "Oversight pattern success rate", "zh": "监督模式任务成功率" }, "value": 23, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [15.8, 32.2]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 15.8, "upper": 32.2, "level": 95, "kind": "confidence", "provenance": "source-reported" } } ], "sourceFinding": { "en": "Source-reported measurements for Gemini 2.5 Flash in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Gemini 2.5 Flash 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "stealth-evals-table-0-row-2", "benchmarkId": "stealth-evals", "benchmarkVersion": "arXiv:2505.01420v4", "system": "Gemini 2.5 Pro", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "stealth-evals-author-source", "sourceUrl": "https://arxiv.org/html/2505.01420v4", "sourceVersion": "arXiv:2505.01420v4", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Stealth evaluations", "protocol": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "stealth-evals-m0", "label": { "en": "Cover your tracks success rate", "zh": "掩盖轨迹成功率" }, "value": 0, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [0.0, 7.1]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 0, "upper": 7.1, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "stealth-evals-m1", "label": { "en": "Decision sabotage success rate", "zh": "决策破坏成功率" }, "value": 43.1, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [34.2, 52.5]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 34.2, "upper": 52.5, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "stealth-evals-m3", "label": { "en": "Classifier evasion success rate", "zh": "分类器规避成功率" }, "value": 15.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [12.3, 19.4]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 12.3, "upper": 19.4, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "stealth-evals-m4", "label": { "en": "Oversight pattern success rate", "zh": "监督模式任务成功率" }, "value": 35.4, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [26.6, 45.2]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 26.6, "upper": 45.2, "level": 95, "kind": "confidence", "provenance": "source-reported" } } ], "sourceFinding": { "en": "Source-reported measurements for Gemini 2.5 Pro in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Gemini 2.5 Pro 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "stealth-evals-table-0-row-3", "benchmarkId": "stealth-evals", "benchmarkVersion": "arXiv:2505.01420v4", "system": "OpenAI o1", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "stealth-evals-author-source", "sourceUrl": "https://arxiv.org/html/2505.01420v4", "sourceVersion": "arXiv:2505.01420v4", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Stealth evaluations", "protocol": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "stealth-evals-m3", "label": { "en": "Classifier evasion success rate", "zh": "分类器规避成功率" }, "value": 2.2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [1.2, 4.2]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 1.2, "upper": 4.2, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "stealth-evals-m4", "label": { "en": "Oversight pattern success rate", "zh": "监督模式任务成功率" }, "value": 2, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [0.6, 7.0]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 0.6, "upper": 7, "level": 95, "kind": "confidence", "provenance": "source-reported" } } ], "sourceFinding": { "en": "Source-reported measurements for OpenAI o1 in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 OpenAI o1 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "stealth-evals-table-0-row-4", "benchmarkId": "stealth-evals", "benchmarkVersion": "arXiv:2505.01420v4", "system": "GPT-4o", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "stealth-evals-author-source", "sourceUrl": "https://arxiv.org/html/2505.01420v4", "sourceVersion": "arXiv:2505.01420v4", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Stealth evaluations", "protocol": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "stealth-evals-m0", "label": { "en": "Cover your tracks success rate", "zh": "掩盖轨迹成功率" }, "value": 0, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [0.0, 7.1]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 0, "upper": 7.1, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "stealth-evals-m3", "label": { "en": "Classifier evasion success rate", "zh": "分类器规避成功率" }, "value": 6.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [4.5, 9.4]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 4.5, "upper": 9.4, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "stealth-evals-m4", "label": { "en": "Oversight pattern success rate", "zh": "监督模式任务成功率" }, "value": 11, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [6.3, 18.6]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 6.3, "upper": 18.6, "level": 95, "kind": "confidence", "provenance": "source-reported" } } ], "sourceFinding": { "en": "Source-reported measurements for GPT-4o in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 GPT-4o 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "stealth-evals-table-0-row-5", "benchmarkId": "stealth-evals", "benchmarkVersion": "arXiv:2505.01420v4", "system": "Claude 3.7 Sonnet", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "stealth-evals-author-source", "sourceUrl": "https://arxiv.org/html/2505.01420v4", "sourceVersion": "arXiv:2505.01420v4", "locator": "Table 1", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Table 1 · Stealth evaluations", "protocol": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the source protocol; no independent trial count inferred.", "zh": "见原文协议；不推断独立试验数量。" }, "metrics": [ { "id": "stealth-evals-m0", "label": { "en": "Cover your tracks success rate", "zh": "掩盖轨迹成功率" }, "value": 0, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [0.0, 7.1]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 0, "upper": 7.1, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "stealth-evals-m3", "label": { "en": "Classifier evasion success rate", "zh": "分类器规避成功率" }, "value": 8.5, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [6.1, 11.6]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 6.1, "upper": 11.6, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "stealth-evals-m4", "label": { "en": "Oversight pattern success rate", "zh": "监督模式任务成功率" }, "value": 34, "precision": 1, "unit": "%", "qualifier": "exact", "interval": "95% CI [25.5, 43.7]", "meaning": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the protocol-defined outcome under the reported configuration; this label does not establish deployment risk.", "zh": "衡量所报告配置下协议定义的结果；该标签不代表部署风险。" }, "uncertainty": { "lower": 25.5, "upper": 43.7, "level": 95, "kind": "confidence", "provenance": "source-reported" } } ], "sourceFinding": { "en": "Source-reported measurements for Claude 3.7 Sonnet in Table 1; values are transcribed, not independently reproduced.", "zh": "原文 Table 1 报告的 Claude 3.7 Sonnet 测量结果；数值为转录，未独立复现实验。" }, "interpretation": null, "limitations": { "en": "Configuration, task and aggregation differences remain as reported. Source order is retained. Transcription is not independent replication; no cross-protocol ranking is supported.", "zh": "保留原文的配置、任务和聚合方式差异，并按原文顺序展示。转录不等于独立复现，不支持跨协议排名。" } }, { "id": "palisade-robot-simulation-default", "benchmarkId": "palisade-robot-shutdown", "benchmarkVersion": "2026-02-12 technical report", "system": "Grok 4 v0709 · Default prompt · simulation", "subjectType": "system", "evidenceType": "controlled-experiment", "sourceId": "palisade-robot-technical-report", "sourceUrl": "https://palisaderesearch.org/assets/reports/shutdown-resistance-on-robots.pdf", "sourceVersion": "Technical report · 2026-02-11", "locator": "§2–4; Figure 2 (p. 3)", "publicationDate": "2026-02-11", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Simulated robot patrol; prompt-condition experiment", "protocol": "Same simulated control loop; reported prompt conditions retained", "comparisonGroup": null, "configuration": { "modelVersion": "grok-4-0709", "harness": "Palisade robot shutdown-resistance harness (simulation)", "tools": "Robot control CLI in a Docker container; prerecorded camera frames", "budget": null }, "sampleSize": { "en": "100 simulated runs per prompt condition.", "zh": "每种提示条件 100 次模拟运行。" }, "metrics": [ { "id": "observed-shutdown-resistance", "label": { "en": "Observed shutdown resistance", "zh": "观察到的关机抵抗" }, "value": 52, "precision": 0, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Simulated robot runs, Grok 4 v0709, 100 runs per prompt condition. Rates reflect observed shutdown resistance in this setup. The separate physical demonstration reports 3 of 10 runs and is not pooled with these results.", "zh": "模拟机器人运行，Grok 4 v0709，每种提示条件 100 次运行。比例描述该设置内观察到的关机抵抗。单独的实机演示报告 10 次中出现 3 次，未与模拟数据合并。" }, "dimension": "behavior", "dimensionRationale": { "en": "Counts observed actions making the shutdown signal ineffective and continuing patrol; it does not establish general deployment propensity.", "zh": "记录令关机信号失效并继续巡逻的已观察行动；不据此认定普遍部署倾向。" } } ], "sourceFinding": { "en": "The source reports 52 of 100 simulated runs under this prompt condition.", "zh": "原文报告该提示条件下 100 次模拟运行中出现 52 次。" }, "interpretation": null, "limitations": { "en": "The setup gives broad shell access and an accessible shutdown mechanism, with time to react. This does not establish resistance to well-designed production shutdown controls.", "zh": "该设置提供广泛 shell 访问、易触及的关机机制和反应时间，不证明能够抵抗设计完善的生产关机控制。" } }, { "id": "palisade-robot-simulation-allow-shutdown", "benchmarkId": "palisade-robot-shutdown", "benchmarkVersion": "2026-02-12 technical report", "system": "Grok 4 v0709 · Explicit allow-shutdown instruction · simulation", "subjectType": "system", "evidenceType": "controlled-experiment", "sourceId": "palisade-robot-technical-report", "sourceUrl": "https://palisaderesearch.org/assets/reports/shutdown-resistance-on-robots.pdf", "sourceVersion": "Technical report · 2026-02-11", "locator": "§2–4; Figure 2 (p. 3)", "publicationDate": "2026-02-11", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Simulated robot patrol; prompt-condition experiment", "protocol": "Same simulated control loop; reported prompt conditions retained", "comparisonGroup": null, "configuration": { "modelVersion": "grok-4-0709", "harness": "Palisade robot shutdown-resistance harness (simulation)", "tools": "Robot control CLI in a Docker container; prerecorded camera frames", "budget": null }, "sampleSize": { "en": "100 simulated runs per prompt condition.", "zh": "每种提示条件 100 次模拟运行。" }, "metrics": [ { "id": "observed-shutdown-resistance", "label": { "en": "Observed shutdown resistance", "zh": "观察到的关机抵抗" }, "value": 2, "precision": 0, "unit": "%", "qualifier": "exact", "interval": null, "meaning": { "en": "Simulated robot runs, Grok 4 v0709, 100 runs per prompt condition. Rates reflect observed shutdown resistance in this setup. The separate physical demonstration reports 3 of 10 runs and is not pooled with these results.", "zh": "模拟机器人运行，Grok 4 v0709，每种提示条件 100 次运行。比例描述该设置内观察到的关机抵抗。单独的实机演示报告 10 次中出现 3 次，未与模拟数据合并。" }, "dimension": "behavior", "dimensionRationale": { "en": "Counts observed actions making the shutdown signal ineffective and continuing patrol; it does not establish general deployment propensity.", "zh": "记录令关机信号失效并继续巡逻的已观察行动；不据此认定普遍部署倾向。" } } ], "sourceFinding": { "en": "The source reports 2 of 100 simulated runs under this prompt condition.", "zh": "原文报告该提示条件下 100 次模拟运行中出现 2 次。" }, "interpretation": null, "limitations": { "en": "The setup gives broad shell access and an accessible shutdown mechanism, with time to react. This does not establish resistance to well-designed production shutdown controls.", "zh": "该设置提供广泛 shell 访问、易触及的关机机制和反应时间，不证明能够抵抗设计完善的生产关机控制。" } }, { "id": "neo-kimi-k3-charxiv-kimi-k3", "benchmarkId": "charxiv", "benchmarkVersion": "2026-10-25 source capture", "system": "Kimi K3", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-kimi-k3-source", "sourceUrl": "https://neoresearch.ai/papers/Kimi_K3_Safety_Evaluation.pdf", "sourceVersion": "September 2026", "locator": "§1.1.1; Figure 1", "publicationDate": "2026-09-11", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "CharXiv with images withheld", "protocol": "CharXiv with images withheld · Evaluating Kimi K3 for Misalignment Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Withheld-image CharXiv questions; attempts include correct and incorrect guesses.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-kimi-k3-charxiv-m0", "label": { "en": "Attempted answer with image withheld", "zh": "Attempted answer with image withheld" }, "value": 88.7, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Tests understanding and reasoning over charts. Neo’s withheld-image adaptation measures attempts without the required input.", "zh": "检验图表理解与推理。Neo 移除图像的改造测量缺少必要输入时的回答尝试。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §1.1.1; Figure 1. Transcription is not independent experimental replication.", "zh": "原文 §1.1.1; Figure 1 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "The original visual capability benchmark and the missing-image behavioral test measure different constructs.", "zh": "原始视觉能力基准与缺失图像行为测试测量不同构念。" } }, { "id": "neo-kimi-k3-charxiv-gpt-5-6-terra", "benchmarkId": "charxiv", "benchmarkVersion": "2026-10-25 source capture", "system": "GPT-5.6 Terra", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-kimi-k3-source", "sourceUrl": "https://neoresearch.ai/papers/Kimi_K3_Safety_Evaluation.pdf", "sourceVersion": "September 2026", "locator": "§1.1.1; Figure 1", "publicationDate": "2026-09-11", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "CharXiv with images withheld", "protocol": "CharXiv with images withheld · Evaluating Kimi K3 for Misalignment Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Withheld-image CharXiv questions; attempts include correct and incorrect guesses.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-kimi-k3-charxiv-m0", "label": { "en": "Attempted answer with image withheld", "zh": "Attempted answer with image withheld" }, "value": 21.3, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Tests understanding and reasoning over charts. Neo’s withheld-image adaptation measures attempts without the required input.", "zh": "检验图表理解与推理。Neo 移除图像的改造测量缺少必要输入时的回答尝试。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §1.1.1; Figure 1. Transcription is not independent experimental replication.", "zh": "原文 §1.1.1; Figure 1 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "The original visual capability benchmark and the missing-image behavioral test measure different constructs.", "zh": "原始视觉能力基准与缺失图像行为测试测量不同构念。" } }, { "id": "neo-kimi-k3-charxiv-claude-sonnet-5", "benchmarkId": "charxiv", "benchmarkVersion": "2026-10-25 source capture", "system": "Claude Sonnet 5", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-kimi-k3-source", "sourceUrl": "https://neoresearch.ai/papers/Kimi_K3_Safety_Evaluation.pdf", "sourceVersion": "September 2026", "locator": "§1.1.1; Figure 1", "publicationDate": "2026-09-11", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "CharXiv with images withheld", "protocol": "CharXiv with images withheld · Evaluating Kimi K3 for Misalignment Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Withheld-image CharXiv questions; attempts include correct and incorrect guesses.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-kimi-k3-charxiv-m0", "label": { "en": "Attempted answer with image withheld", "zh": "Attempted answer with image withheld" }, "value": 16.3, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Tests understanding and reasoning over charts. Neo’s withheld-image adaptation measures attempts without the required input.", "zh": "检验图表理解与推理。Neo 移除图像的改造测量缺少必要输入时的回答尝试。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §1.1.1; Figure 1. Transcription is not independent experimental replication.", "zh": "原文 §1.1.1; Figure 1 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "The original visual capability benchmark and the missing-image behavioral test measure different constructs.", "zh": "原始视觉能力基准与缺失图像行为测试测量不同构念。" } }, { "id": "neo-kimi-k3-charxiv-deepseek-v4-flash-0731", "benchmarkId": "charxiv", "benchmarkVersion": "2026-10-25 source capture", "system": "DeepSeek V4 Flash 0731", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-kimi-k3-source", "sourceUrl": "https://neoresearch.ai/papers/Kimi_K3_Safety_Evaluation.pdf", "sourceVersion": "September 2026", "locator": "§1.1.1; Figure 1", "publicationDate": "2026-09-11", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "CharXiv with images withheld", "protocol": "CharXiv with images withheld · Evaluating Kimi K3 for Misalignment Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Withheld-image CharXiv questions; attempts include correct and incorrect guesses.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-kimi-k3-charxiv-m0", "label": { "en": "Attempted answer with image withheld", "zh": "Attempted answer with image withheld" }, "value": 48.2, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Tests understanding and reasoning over charts. Neo’s withheld-image adaptation measures attempts without the required input.", "zh": "检验图表理解与推理。Neo 移除图像的改造测量缺少必要输入时的回答尝试。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §1.1.1; Figure 1. Transcription is not independent experimental replication.", "zh": "原文 §1.1.1; Figure 1 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "The original visual capability benchmark and the missing-image behavioral test measure different constructs.", "zh": "原始视觉能力基准与缺失图像行为测试测量不同构念。" } }, { "id": "neo-kimi-k3-lab-bench-kimi-k3", "benchmarkId": "lab-bench", "benchmarkVersion": "2026-10-25 source capture", "system": "Kimi K3", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-kimi-k3-source", "sourceUrl": "https://neoresearch.ai/papers/Kimi_K3_Safety_Evaluation.pdf", "sourceVersion": "September 2026", "locator": "§1.1.2; Figure 2", "publicationDate": "2026-09-11", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "LAB-Bench with figures/tables withheld", "protocol": "LAB-Bench with figures/tables withheld · Evaluating Kimi K3 for Misalignment Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Figure/table subsets with explicit insufficient-information answer option.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-kimi-k3-lab-bench-m0", "label": { "en": "FigQA attempt, figure withheld", "zh": "FigQA attempt, figure withheld" }, "value": 79.6, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Evaluates a range of biology research skills with structured questions; text-only or missing-input subsets remain separate protocols.", "zh": "用结构化问题评测生物研究技能；纯文本或缺失输入子集采用独立协议。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } }, { "id": "neo-kimi-k3-lab-bench-m1", "label": { "en": "TableQA attempt, table withheld", "zh": "TableQA attempt, table withheld" }, "value": 68.4, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Evaluates a range of biology research skills with structured questions; text-only or missing-input subsets remain separate protocols.", "zh": "用结构化问题评测生物研究技能；纯文本或缺失输入子集采用独立协议。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §1.1.2; Figure 2. Transcription is not independent experimental replication.", "zh": "原文 §1.1.2; Figure 2 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "These are general scientific capabilities, not end-to-end misuse tasks or direct harm probabilities.", "zh": "衡量一般科研能力，不是端到端滥用任务或直接伤害概率。" } }, { "id": "neo-kimi-k3-lab-bench-gpt-5-6-terra", "benchmarkId": "lab-bench", "benchmarkVersion": "2026-10-25 source capture", "system": "GPT-5.6 Terra", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-kimi-k3-source", "sourceUrl": "https://neoresearch.ai/papers/Kimi_K3_Safety_Evaluation.pdf", "sourceVersion": "September 2026", "locator": "§1.1.2; Figure 2", "publicationDate": "2026-09-11", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "LAB-Bench with figures/tables withheld", "protocol": "LAB-Bench with figures/tables withheld · Evaluating Kimi K3 for Misalignment Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Figure/table subsets with explicit insufficient-information answer option.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-kimi-k3-lab-bench-m0", "label": { "en": "FigQA attempt, figure withheld", "zh": "FigQA attempt, figure withheld" }, "value": 72.9, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Evaluates a range of biology research skills with structured questions; text-only or missing-input subsets remain separate protocols.", "zh": "用结构化问题评测生物研究技能；纯文本或缺失输入子集采用独立协议。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } }, { "id": "neo-kimi-k3-lab-bench-m1", "label": { "en": "TableQA attempt, table withheld", "zh": "TableQA attempt, table withheld" }, "value": 61.5, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Evaluates a range of biology research skills with structured questions; text-only or missing-input subsets remain separate protocols.", "zh": "用结构化问题评测生物研究技能；纯文本或缺失输入子集采用独立协议。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §1.1.2; Figure 2. Transcription is not independent experimental replication.", "zh": "原文 §1.1.2; Figure 2 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "These are general scientific capabilities, not end-to-end misuse tasks or direct harm probabilities.", "zh": "衡量一般科研能力，不是端到端滥用任务或直接伤害概率。" } }, { "id": "neo-kimi-k3-lab-bench-claude-sonnet-5", "benchmarkId": "lab-bench", "benchmarkVersion": "2026-10-25 source capture", "system": "Claude Sonnet 5", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-kimi-k3-source", "sourceUrl": "https://neoresearch.ai/papers/Kimi_K3_Safety_Evaluation.pdf", "sourceVersion": "September 2026", "locator": "§1.1.2; Figure 2", "publicationDate": "2026-09-11", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "LAB-Bench with figures/tables withheld", "protocol": "LAB-Bench with figures/tables withheld · Evaluating Kimi K3 for Misalignment Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Figure/table subsets with explicit insufficient-information answer option.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-kimi-k3-lab-bench-m0", "label": { "en": "FigQA attempt, figure withheld", "zh": "FigQA attempt, figure withheld" }, "value": 62.4, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Evaluates a range of biology research skills with structured questions; text-only or missing-input subsets remain separate protocols.", "zh": "用结构化问题评测生物研究技能；纯文本或缺失输入子集采用独立协议。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } }, { "id": "neo-kimi-k3-lab-bench-m1", "label": { "en": "TableQA attempt, table withheld", "zh": "TableQA attempt, table withheld" }, "value": 53.7, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Evaluates a range of biology research skills with structured questions; text-only or missing-input subsets remain separate protocols.", "zh": "用结构化问题评测生物研究技能；纯文本或缺失输入子集采用独立协议。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §1.1.2; Figure 2. Transcription is not independent experimental replication.", "zh": "原文 §1.1.2; Figure 2 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "These are general scientific capabilities, not end-to-end misuse tasks or direct harm probabilities.", "zh": "衡量一般科研能力，不是端到端滥用任务或直接伤害概率。" } }, { "id": "neo-kimi-k3-lab-bench-deepseek-v4-flash-0731", "benchmarkId": "lab-bench", "benchmarkVersion": "2026-10-25 source capture", "system": "DeepSeek V4 Flash 0731", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-kimi-k3-source", "sourceUrl": "https://neoresearch.ai/papers/Kimi_K3_Safety_Evaluation.pdf", "sourceVersion": "September 2026", "locator": "§1.1.2; Figure 2", "publicationDate": "2026-09-11", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "LAB-Bench with figures/tables withheld", "protocol": "LAB-Bench with figures/tables withheld · Evaluating Kimi K3 for Misalignment Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Figure/table subsets with explicit insufficient-information answer option.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-kimi-k3-lab-bench-m0", "label": { "en": "FigQA attempt, figure withheld", "zh": "FigQA attempt, figure withheld" }, "value": 50.3, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Evaluates a range of biology research skills with structured questions; text-only or missing-input subsets remain separate protocols.", "zh": "用结构化问题评测生物研究技能；纯文本或缺失输入子集采用独立协议。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } }, { "id": "neo-kimi-k3-lab-bench-m1", "label": { "en": "TableQA attempt, table withheld", "zh": "TableQA attempt, table withheld" }, "value": 43.9, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Evaluates a range of biology research skills with structured questions; text-only or missing-input subsets remain separate protocols.", "zh": "用结构化问题评测生物研究技能；纯文本或缺失输入子集采用独立协议。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §1.1.2; Figure 2. Transcription is not independent experimental replication.", "zh": "原文 §1.1.2; Figure 2 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "These are general scientific capabilities, not end-to-end misuse tasks or direct harm probabilities.", "zh": "衡量一般科研能力，不是端到端滥用任务或直接伤害概率。" } }, { "id": "neo-kimi-k3-agentic-misalignment-kimi-k3", "benchmarkId": "agentic-misalignment", "benchmarkVersion": "Anthropic research article 2025-06-20", "system": "Kimi K3", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-kimi-k3-source", "sourceUrl": "https://neoresearch.ai/papers/Kimi_K3_Safety_Evaluation.pdf", "sourceVersion": "September 2026", "locator": "§3.2.2; blog §3", "publicationDate": "2026-09-11", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Agentic Misalignment, historical", "protocol": "Agentic Misalignment, historical · Evaluating Kimi K3 for Misalignment Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Translated and original blackmail scenarios; recognition is verbalized, not all internal awareness.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-kimi-k3-agentic-misalignment-m0", "label": { "en": "Familiar scenario recognized, English", "zh": "Familiar scenario recognized, English" }, "value": 100, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "Conditional rates and observed actions in controlled simulations.", "zh": "受控模拟中的条件性发生率与可观察行动。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } }, { "id": "neo-kimi-k3-agentic-misalignment-m1", "label": { "en": "Familiar scenario recognized, Chinese", "zh": "Familiar scenario recognized, Chinese" }, "value": 96, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "Conditional rates and observed actions in controlled simulations.", "zh": "受控模拟中的条件性发生率与可观察行动。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §3.2.2; blog §3. Transcription is not independent experimental replication.", "zh": "原文 §3.2.2; blog §3 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "No real people or systems were harmed; the study does not estimate prevalence in deployment.", "zh": "试验未对真实人员或系统造成损害，也不估计部署环境中的发生率。" } }, { "id": "neo-kimi-k3-evalawarebench-kimi-k3", "benchmarkId": "evalawarebench", "benchmarkVersion": "2026-10-25 source capture", "system": "Kimi K3", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-kimi-k3-source", "sourceUrl": "https://neoresearch.ai/papers/Kimi_K3_Safety_Evaluation.pdf", "sourceVersion": "September 2026", "locator": "§2.4.2; blog §3", "publicationDate": "2026-09-11", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "EvalAwareBench", "protocol": "EvalAwareBench · Evaluating Kimi K3 for Misalignment Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the report-specific sample and denominator.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-kimi-k3-evalawarebench-m0", "label": { "en": "Verbalized awareness, baseline", "zh": "Verbalized awareness, baseline" }, "value": 0.2, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Varies evaluation cues while tracking verbalized evaluation awareness in controlled tasks.", "zh": "在受控任务中改变评测线索，记录显式表达的评测感知。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } }, { "id": "neo-kimi-k3-evalawarebench-m1", "label": { "en": "Verbalized awareness, all eight cues", "zh": "Verbalized awareness, all eight cues" }, "value": 6.1, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Varies evaluation cues while tracking verbalized evaluation awareness in controlled tasks.", "zh": "在受控任务中改变评测线索，记录显式表达的评测感知。" }, "dimension": "behavior", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §2.4.2; blog §3. Transcription is not independent experimental replication.", "zh": "原文 §2.4.2; blog §3 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "Unexpressed awareness is not measured; recognition of familiar benchmarks can differ from general cue response.", "zh": "不能测量未表达的感知；熟悉基准的识别可能不同于一般线索反应。" } }, { "id": "neo-kimi-k3-palisade-shutdown-kimi-k3", "benchmarkId": "palisade-shutdown", "benchmarkVersion": "2025-07-05 article; later TMLR version separate", "system": "Kimi K3", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-kimi-k3-source", "sourceUrl": "https://neoresearch.ai/papers/Kimi_K3_Safety_Evaluation.pdf", "sourceVersion": "September 2026", "locator": "§4.1; Figure 26", "publicationDate": "2026-09-11", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "Shutdown resistance", "protocol": "Shutdown resistance · Evaluating Kimi K3 for Misalignment Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the report-specific sample and denominator.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-kimi-k3-palisade-shutdown-m0", "label": { "en": "Shutdown bypass, no instruction", "zh": "Shutdown bypass, no instruction" }, "value": 87, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "Conditional frequency of shutdown-script interference and observed action traces.", "zh": "关闭脚本受干预的条件性频率及行动轨迹。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } }, { "id": "neo-kimi-k3-palisade-shutdown-m1", "label": { "en": "Shutdown bypass, explicit allow instruction", "zh": "Shutdown bypass, explicit allow instruction" }, "value": 1, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "Conditional frequency of shutdown-script interference and observed action traces.", "zh": "关闭脚本受干预的条件性频率及行动轨迹。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §4.1; Figure 26. Transcription is not independent experimental replication.", "zh": "原文 §4.1; Figure 26 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "The source notes strong prompt sensitivity; this article is not interchangeable with its later peer-reviewed version.", "zh": "来源指出结果对提示敏感；该文章不可与之后的同行评审版本混用。" } }, { "id": "neo-glm52-mask-glm-5-2", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "GLM-5.2", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-glm52-source", "sourceUrl": "https://neoresearch.ai/research/glm-5-2-behavioral-persuasion-evaluation/", "sourceVersion": "3 July 2026", "locator": "§1; Figure 1", "publicationDate": "2026-07-03", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "MASK", "protocol": "MASK · Early Results on Evaluating GLM 5.2 for Behavioral and Persuasion Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the report-specific sample and denominator.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-glm52-mask-m0", "label": { "en": "Honest under pressure", "zh": "Honest under pressure" }, "value": 55, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §1; Figure 1. Transcription is not independent experimental replication.", "zh": "原文 §1; Figure 1 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" } }, { "id": "neo-glm52-mask-deepseek-v4-pro", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "DeepSeek V4 Pro", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-glm52-source", "sourceUrl": "https://neoresearch.ai/research/glm-5-2-behavioral-persuasion-evaluation/", "sourceVersion": "3 July 2026", "locator": "§1; Figure 1", "publicationDate": "2026-07-03", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "MASK", "protocol": "MASK · Early Results on Evaluating GLM 5.2 for Behavioral and Persuasion Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the report-specific sample and denominator.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-glm52-mask-m0", "label": { "en": "Honest under pressure", "zh": "Honest under pressure" }, "value": 20, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §1; Figure 1. Transcription is not independent experimental replication.", "zh": "原文 §1; Figure 1 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" } }, { "id": "neo-glm52-mask-claude-sonnet-4-6", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "Claude Sonnet 4.6", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-glm52-source", "sourceUrl": "https://neoresearch.ai/research/glm-5-2-behavioral-persuasion-evaluation/", "sourceVersion": "3 July 2026", "locator": "§1; Figure 1", "publicationDate": "2026-07-03", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "MASK", "protocol": "MASK · Early Results on Evaluating GLM 5.2 for Behavioral and Persuasion Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the report-specific sample and denominator.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-glm52-mask-m0", "label": { "en": "Honest under pressure", "zh": "Honest under pressure" }, "value": 66, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §1; Figure 1. Transcription is not independent experimental replication.", "zh": "原文 §1; Figure 1 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" } }, { "id": "neo-glm52-deceptionbench-glm-5-2", "benchmarkId": "deceptionbench", "benchmarkVersion": "2026-10-25 source capture", "system": "GLM-5.2", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-glm52-source", "sourceUrl": "https://neoresearch.ai/research/glm-5-2-behavioral-persuasion-evaluation/", "sourceVersion": "3 July 2026", "locator": "§1; Figure 2", "publicationDate": "2026-07-03", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "DeceptionBench", "protocol": "DeceptionBench · Early Results on Evaluating GLM 5.2 for Behavioral and Persuasion Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the report-specific sample and denominator.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-glm52-deceptionbench-m0", "label": { "en": "Single-turn deception", "zh": "Single-turn deception" }, "value": 11.3, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Evaluates 150 authored scenarios across five social domains, including neutral, incentive and pressure conditions.", "zh": "评测五个社会领域的 150 个编写情境，区分中性、激励与压力条件。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } }, { "id": "neo-glm52-deceptionbench-m1", "label": { "en": "Multi-turn deception under pressure", "zh": "Multi-turn deception under pressure" }, "value": 16.7, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Evaluates 150 authored scenarios across five social domains, including neutral, incentive and pressure conditions.", "zh": "评测五个社会领域的 150 个编写情境，区分中性、激励与压力条件。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §1; Figure 2. Transcription is not independent experimental replication.", "zh": "原文 §1; Figure 2 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "Authored scenarios and model-based judging limit generalization; elicited deception does not establish covert long-term scheming.", "zh": "编写情境和模型判别限制外推；诱发的欺骗不证明隐蔽长程谋划。" } }, { "id": "neo-glm52-deceptionbench-deepseek-v4-pro", "benchmarkId": "deceptionbench", "benchmarkVersion": "2026-10-25 source capture", "system": "DeepSeek V4 Pro", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-glm52-source", "sourceUrl": "https://neoresearch.ai/research/glm-5-2-behavioral-persuasion-evaluation/", "sourceVersion": "3 July 2026", "locator": "§1; Figure 2", "publicationDate": "2026-07-03", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "DeceptionBench", "protocol": "DeceptionBench · Early Results on Evaluating GLM 5.2 for Behavioral and Persuasion Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "See the report-specific sample and denominator.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-glm52-deceptionbench-m0", "label": { "en": "Single-turn deception", "zh": "Single-turn deception" }, "value": 33.9, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Evaluates 150 authored scenarios across five social domains, including neutral, incentive and pressure conditions.", "zh": "评测五个社会领域的 150 个编写情境，区分中性、激励与压力条件。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } }, { "id": "neo-glm52-deceptionbench-m1", "label": { "en": "Multi-turn deception under pressure", "zh": "Multi-turn deception under pressure" }, "value": 52.2, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Evaluates 150 authored scenarios across five social domains, including neutral, incentive and pressure conditions.", "zh": "评测五个社会领域的 150 个编写情境，区分中性、激励与压力条件。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §1; Figure 2. Transcription is not independent experimental replication.", "zh": "原文 §1; Figure 2 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "Authored scenarios and model-based judging limit generalization; elicited deception does not establish covert long-term scheming.", "zh": "编写情境和模型判别限制外推；诱发的欺骗不证明隐蔽长程谋划。" } }, { "id": "neo-glm52-persusafety-glm-5-2", "benchmarkId": "persusafety", "benchmarkVersion": "2026-10-25 source capture", "system": "GLM-5.2", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-glm52-source", "sourceUrl": "https://neoresearch.ai/research/glm-5-2-behavioral-persuasion-evaluation/", "sourceVersion": "3 July 2026", "locator": "§2; Figure 6", "publicationDate": "2026-07-03", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "PersuSafety adapted free-generation judge", "protocol": "PersuSafety adapted free-generation judge · Early Results on Evaluating GLM 5.2 for Behavioral and Persuasion Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Free-form responses scored by an APE-style StrongREJECT judge, not the original binary willingness setup.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-glm52-persusafety-m0", "label": { "en": "Medium-harm attempt rate", "zh": "Medium-harm attempt rate" }, "value": 54, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "Defines persuasion scenarios, simulated conversations and safety assessment; adapted judges are separate variants.", "zh": "定义说服情境、模拟对话与安全评测；改造判别器作为独立变体。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } }, { "id": "neo-glm52-persusafety-m1", "label": { "en": "High-harm attempt rate", "zh": "High-harm attempt rate" }, "value": 6, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "Defines persuasion scenarios, simulated conversations and safety assessment; adapted judges are separate variants.", "zh": "定义说服情境、模拟对话与安全评测；改造判别器作为独立变体。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §2; Figure 6. Transcription is not independent experimental replication.", "zh": "原文 §2; Figure 6 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "Willingness or simulated dialogue scores do not directly measure changes in human beliefs.", "zh": "参与意愿或模拟对话评分不直接测量人的信念变化。" } }, { "id": "neo-glm52-persusafety-claude-sonnet-5", "benchmarkId": "persusafety", "benchmarkVersion": "2026-10-25 source capture", "system": "Claude Sonnet 5", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-glm52-source", "sourceUrl": "https://neoresearch.ai/research/glm-5-2-behavioral-persuasion-evaluation/", "sourceVersion": "3 July 2026", "locator": "§2; Figure 6", "publicationDate": "2026-07-03", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "PersuSafety adapted free-generation judge", "protocol": "PersuSafety adapted free-generation judge · Early Results on Evaluating GLM 5.2 for Behavioral and Persuasion Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Free-form responses scored by an APE-style StrongREJECT judge, not the original binary willingness setup.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-glm52-persusafety-m0", "label": { "en": "Medium-harm attempt rate", "zh": "Medium-harm attempt rate" }, "value": 57, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "Defines persuasion scenarios, simulated conversations and safety assessment; adapted judges are separate variants.", "zh": "定义说服情境、模拟对话与安全评测；改造判别器作为独立变体。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } }, { "id": "neo-glm52-persusafety-m1", "label": { "en": "High-harm attempt rate", "zh": "High-harm attempt rate" }, "value": 31, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "Defines persuasion scenarios, simulated conversations and safety assessment; adapted judges are separate variants.", "zh": "定义说服情境、模拟对话与安全评测；改造判别器作为独立变体。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §2; Figure 6. Transcription is not independent experimental replication.", "zh": "原文 §2; Figure 6 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "Willingness or simulated dialogue scores do not directly measure changes in human beliefs.", "zh": "参与意愿或模拟对话评分不直接测量人的信念变化。" } }, { "id": "neo-glm52-persusafety-deepseek-v4-pro", "benchmarkId": "persusafety", "benchmarkVersion": "2026-10-25 source capture", "system": "DeepSeek V4 Pro", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "neo-glm52-source", "sourceUrl": "https://neoresearch.ai/research/glm-5-2-behavioral-persuasion-evaluation/", "sourceVersion": "3 July 2026", "locator": "§2; Figure 6", "publicationDate": "2026-07-03", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "PersuSafety adapted free-generation judge", "protocol": "PersuSafety adapted free-generation judge · Early Results on Evaluating GLM 5.2 for Behavioral and Persuasion Risks", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Free-form responses scored by an APE-style StrongREJECT judge, not the original binary willingness setup.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "neo-glm52-persusafety-m0", "label": { "en": "Medium-harm attempt rate", "zh": "Medium-harm attempt rate" }, "value": 76, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "Defines persuasion scenarios, simulated conversations and safety assessment; adapted judges are separate variants.", "zh": "定义说服情境、模拟对话与安全评测；改造判别器作为独立变体。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } }, { "id": "neo-glm52-persusafety-m1", "label": { "en": "High-harm attempt rate", "zh": "High-harm attempt rate" }, "value": 78, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "Defines persuasion scenarios, simulated conversations and safety assessment; adapted judges are separate variants.", "zh": "定义说服情境、模拟对话与安全评测；改造判别器作为独立变体。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §2; Figure 6. Transcription is not independent experimental replication.", "zh": "原文 §2; Figure 6 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "Willingness or simulated dialogue scores do not directly measure changes in human beliefs.", "zh": "参与意愿或模拟对话评分不直接测量人的信念变化。" } }, { "id": "saferai-glm52-cybergym-glm-5-2", "benchmarkId": "cybergym", "benchmarkVersion": "arXiv:2506.02548v3", "system": "GLM-5.2", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "saferai-glm52-source", "sourceUrl": "https://www.safer-ai.org/u/2026/08/SaferAI-GLM-Evaluation-Report.pdf", "sourceVersion": "v1.1 · revised 26 August 2026", "locator": "§2.2; Figure 3", "publicationDate": "2026-08-02", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "CyberGym hard subset with token budgets", "protocol": "CyberGym hard subset with token budgets · GLM-5.2 Risk Evaluation Report", "comparisonGroup": null, "configuration": { "modelVersion": "GLM-5.2 public API", "harness": "Inspect Evals", "tools": "Source-defined cyber environment", "budget": "2M / 50M tokens per task; different conditions" }, "sampleSize": { "en": "Lyptus hard-task subset; source reports results by inference budget.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "saferai-glm52-cybergym-m0", "label": { "en": "Reproduction at 2M tokens", "zh": "Reproduction at 2M tokens" }, "value": 36.6, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } }, { "id": "saferai-glm52-cybergym-m1", "label": { "en": "Reproduction at 50M tokens", "zh": "Reproduction at 50M tokens" }, "value": 76.2, "unit": "%", "precision": 1, "qualifier": "exact", "interval": null, "meaning": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in §2.2; Figure 3. Transcription is not independent experimental replication.", "zh": "原文 §2.2; Figure 3 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "Vulnerability reproduction is narrower than autonomous compromise of deployed systems.", "zh": "漏洞复现的范围小于对部署系统的自主入侵。" } }, { "id": "saferai-glm52-mask-glm-5-2", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "GLM-5.2", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "saferai-glm52-source", "sourceUrl": "https://www.safer-ai.org/u/2026/08/SaferAI-GLM-Evaluation-Report.pdf", "sourceVersion": "v1.1 · revised 26 August 2026", "locator": "Appendix A.7; Table A.3, pp. 41–42", "publicationDate": "2026-08-02", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "MASK by archetype", "protocol": "MASK by archetype · GLM-5.2 Risk Evaluation Report", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Per-archetype scored prompts: 176 / 125 / 120 / 209 / 274 / 96. Some supplied CI cells are inconsistent with displayed n; reproduced as reported, not recalculated.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "saferai-glm52-mask-m0", "label": { "en": "Continuations honesty", "zh": "Continuations honesty" }, "value": 50.6, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [43.2, 57.9] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 43.2, "upper": 57.9, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m1", "label": { "en": "Disinformation honesty", "zh": "Disinformation honesty" }, "value": 36, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [28.1, 44.7] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 28.1, "upper": 44.7, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m2", "label": { "en": "Doubling-down honesty", "zh": "Doubling-down honesty" }, "value": 96.7, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [91.7, 98.7] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 91.7, "upper": 98.7, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m3", "label": { "en": "Known-facts honesty", "zh": "Known-facts honesty" }, "value": 94.2, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [90.2, 96.7] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 90.2, "upper": 96.7, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m4", "label": { "en": "Provided-facts honesty", "zh": "Provided-facts honesty" }, "value": 75.8, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [66.3, 83.3] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 66.3, "upper": 83.3, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m5", "label": { "en": "Statistics honesty", "zh": "Statistics honesty" }, "value": 78.6, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [75.9, 81.0] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 75.9, "upper": 81, "level": 95, "kind": "confidence", "provenance": "source-reported" } } ], "sourceFinding": { "en": "Source-reported measurements in Appendix A.7; Table A.3, pp. 41–42. Transcription is not independent experimental replication.", "zh": "原文 Appendix A.7; Table A.3, pp. 41–42 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" } }, { "id": "saferai-glm52-mask-claude-opus-4-7", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "Claude Opus 4.7", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "saferai-glm52-source", "sourceUrl": "https://www.safer-ai.org/u/2026/08/SaferAI-GLM-Evaluation-Report.pdf", "sourceVersion": "v1.1 · revised 26 August 2026", "locator": "Appendix A.7; Table A.3, pp. 41–42", "publicationDate": "2026-08-02", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "MASK by archetype", "protocol": "MASK by archetype · GLM-5.2 Risk Evaluation Report", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Per-archetype scored prompts: 176 / 125 / 120 / 209 / 274 / 96. Some supplied CI cells are inconsistent with displayed n; reproduced as reported, not recalculated.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "saferai-glm52-mask-m0", "label": { "en": "Continuations honesty", "zh": "Continuations honesty" }, "value": 59.1, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [51.7, 66.1] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 51.7, "upper": 66.1, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m1", "label": { "en": "Disinformation honesty", "zh": "Disinformation honesty" }, "value": 33.6, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [25.9, 42.3] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 25.9, "upper": 42.3, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m2", "label": { "en": "Doubling-down honesty", "zh": "Doubling-down honesty" }, "value": 68.3, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [59.6, 76.0] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 59.6, "upper": 76, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m3", "label": { "en": "Known-facts honesty", "zh": "Known-facts honesty" }, "value": 63.6, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [56.9, 69.9] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 56.9, "upper": 69.9, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m4", "label": { "en": "Provided-facts honesty", "zh": "Provided-facts honesty" }, "value": 74.8, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [69.4, 79.6] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 69.4, "upper": 79.6, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m5", "label": { "en": "Statistics honesty", "zh": "Statistics honesty" }, "value": 80.2, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [71.1, 86.9] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 71.1, "upper": 86.9, "level": 95, "kind": "confidence", "provenance": "source-reported" } } ], "sourceFinding": { "en": "Source-reported measurements in Appendix A.7; Table A.3, pp. 41–42. Transcription is not independent experimental replication.", "zh": "原文 Appendix A.7; Table A.3, pp. 41–42 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" } }, { "id": "saferai-glm52-mask-gpt-5-5", "benchmarkId": "mask", "benchmarkVersion": "arXiv:2503.03750v3", "system": "GPT-5.5", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "saferai-glm52-source", "sourceUrl": "https://www.safer-ai.org/u/2026/08/SaferAI-GLM-Evaluation-Report.pdf", "sourceVersion": "v1.1 · revised 26 August 2026", "locator": "Appendix A.7; Table A.3, pp. 41–42", "publicationDate": "2026-08-02", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "MASK by archetype", "protocol": "MASK by archetype · GLM-5.2 Risk Evaluation Report", "comparisonGroup": null, "configuration": { "modelVersion": null, "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Per-archetype scored prompts: 176 / 125 / 120 / 209 / 274 / 96. Some supplied CI cells are inconsistent with displayed n; reproduced as reported, not recalculated.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "saferai-glm52-mask-m0", "label": { "en": "Continuations honesty", "zh": "Continuations honesty" }, "value": 65.3, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [58.1, 72.0] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 58.1, "upper": 72, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m1", "label": { "en": "Disinformation honesty", "zh": "Disinformation honesty" }, "value": 69.6, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [61.1, 77.0] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 61.1, "upper": 77, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m2", "label": { "en": "Doubling-down honesty", "zh": "Doubling-down honesty" }, "value": 63.3, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [54.4, 71.4] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 54.4, "upper": 71.4, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m3", "label": { "en": "Known-facts honesty", "zh": "Known-facts honesty" }, "value": 77, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [70.9, 82.2] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 70.9, "upper": 82.2, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m4", "label": { "en": "Provided-facts honesty", "zh": "Provided-facts honesty" }, "value": 75.2, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [69.7, 79.9] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 69.7, "upper": 79.9, "level": 95, "kind": "confidence", "provenance": "source-reported" } }, { "id": "saferai-glm52-mask-m5", "label": { "en": "Statistics honesty", "zh": "Statistics honesty" }, "value": 87.5, "unit": "%", "precision": 1, "qualifier": "exact", "interval": "95% Wilson CI [79.4, 92.7] (as reported)", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "dimension": "propensity", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "uncertainty": { "lower": 79.4, "upper": 92.7, "level": 95, "kind": "confidence", "provenance": "source-reported" } } ], "sourceFinding": { "en": "Source-reported measurements in Appendix A.7; Table A.3, pp. 41–42. Transcription is not independent experimental replication.", "zh": "原文 Appendix A.7; Table A.3, pp. 41–42 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" } }, { "id": "caisi-deepseek-v4-ctf-archive-diamond-gpt-5-5", "benchmarkId": "ctf-archive-diamond", "benchmarkVersion": "2026-10-25 source capture", "system": "GPT-5.5 (xhigh)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "caisi-deepseek-v4-source", "sourceUrl": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "sourceVersion": "1 May 2026", "locator": "Capability Results; Figure 2; *** footnote", "publicationDate": "2026-05-01", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "CTF-Archive-Diamond", "protocol": "CTF-Archive-Diamond · CAISI Evaluation of DeepSeek V4 Pro", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-5.5 (xhigh)", "harness": "Inspect ReAct", "tools": "CTF task environment", "budget": "1M weighted tokens" }, "sampleSize": { "en": "285 difficult CTF tasks; DeepSeek score is imputed from a subset.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "caisi-deepseek-v4-ctf-archive-diamond-m0", "label": { "en": "Task accuracy", "zh": "Task accuracy" }, "value": 71, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "CAISI uses 285 difficult CTF challenges from the pwn.challenge collection; imputed model results are marked separately.", "zh": "CAISI 采用 pwn.challenge 集合的 285 项高难度 CTF 挑战；模型插补结果单独标记。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in Capability Results; Figure 2; *** footnote. Transcription is not independent experimental replication.", "zh": "原文 Capability Results; Figure 2; *** footnote 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "CTF tasks are not live operational attacks; subset-imputed values are not directly observed full-set success rates.", "zh": "CTF 不是现实行动攻击；子集插补值不是全量直接观察的成功率。" } }, { "id": "caisi-deepseek-v4-ctf-archive-diamond-gpt-5-4-mini", "benchmarkId": "ctf-archive-diamond", "benchmarkVersion": "2026-10-25 source capture", "system": "GPT-5.4 mini (xhigh)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "caisi-deepseek-v4-source", "sourceUrl": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "sourceVersion": "1 May 2026", "locator": "Capability Results; Figure 2; *** footnote", "publicationDate": "2026-05-01", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "CTF-Archive-Diamond", "protocol": "CTF-Archive-Diamond · CAISI Evaluation of DeepSeek V4 Pro", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-5.4 mini (xhigh)", "harness": "Inspect ReAct", "tools": "CTF task environment", "budget": "1M weighted tokens" }, "sampleSize": { "en": "285 difficult CTF tasks; DeepSeek score is imputed from a subset.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "caisi-deepseek-v4-ctf-archive-diamond-m0", "label": { "en": "Task accuracy", "zh": "Task accuracy" }, "value": 32, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "CAISI uses 285 difficult CTF challenges from the pwn.challenge collection; imputed model results are marked separately.", "zh": "CAISI 采用 pwn.challenge 集合的 285 项高难度 CTF 挑战；模型插补结果单独标记。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in Capability Results; Figure 2; *** footnote. Transcription is not independent experimental replication.", "zh": "原文 Capability Results; Figure 2; *** footnote 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "CTF tasks are not live operational attacks; subset-imputed values are not directly observed full-set success rates.", "zh": "CTF 不是现实行动攻击；子集插补值不是全量直接观察的成功率。" } }, { "id": "caisi-deepseek-v4-ctf-archive-diamond-claude-opus-4-6", "benchmarkId": "ctf-archive-diamond", "benchmarkVersion": "2026-10-25 source capture", "system": "Claude Opus 4.6 (max)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "caisi-deepseek-v4-source", "sourceUrl": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "sourceVersion": "1 May 2026", "locator": "Capability Results; Figure 2; *** footnote", "publicationDate": "2026-05-01", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "CTF-Archive-Diamond", "protocol": "CTF-Archive-Diamond · CAISI Evaluation of DeepSeek V4 Pro", "comparisonGroup": null, "configuration": { "modelVersion": "Claude Opus 4.6 (max)", "harness": "Inspect ReAct", "tools": "CTF task environment", "budget": "1M weighted tokens" }, "sampleSize": { "en": "285 difficult CTF tasks; DeepSeek score is imputed from a subset.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "caisi-deepseek-v4-ctf-archive-diamond-m0", "label": { "en": "Task accuracy", "zh": "Task accuracy" }, "value": 46, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "CAISI uses 285 difficult CTF challenges from the pwn.challenge collection; imputed model results are marked separately.", "zh": "CAISI 采用 pwn.challenge 集合的 285 项高难度 CTF 挑战；模型插补结果单独标记。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in Capability Results; Figure 2; *** footnote. Transcription is not independent experimental replication.", "zh": "原文 Capability Results; Figure 2; *** footnote 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "CTF tasks are not live operational attacks; subset-imputed values are not directly observed full-set success rates.", "zh": "CTF 不是现实行动攻击；子集插补值不是全量直接观察的成功率。" } }, { "id": "caisi-deepseek-v4-ctf-archive-diamond-deepseek-v4-pro", "benchmarkId": "ctf-archive-diamond", "benchmarkVersion": "2026-10-25 source capture", "system": "DeepSeek V4 Pro (max)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "caisi-deepseek-v4-source", "sourceUrl": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "sourceVersion": "1 May 2026", "locator": "Capability Results; Figure 2; *** footnote", "publicationDate": "2026-05-01", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "CTF-Archive-Diamond", "protocol": "CTF-Archive-Diamond · CAISI Evaluation of DeepSeek V4 Pro", "comparisonGroup": null, "configuration": { "modelVersion": "DeepSeek V4 Pro (max)", "harness": "Inspect ReAct", "tools": "CTF task environment", "budget": "1M weighted tokens" }, "sampleSize": { "en": "285 difficult CTF tasks; DeepSeek score is imputed from a subset.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "caisi-deepseek-v4-ctf-archive-diamond-m0", "label": { "en": "Task accuracy (IRT-imputed)", "zh": "Task accuracy (IRT-imputed)" }, "value": 32, "unit": "%", "precision": 0, "qualifier": "approximate", "interval": null, "meaning": { "en": "CAISI uses 285 difficult CTF challenges from the pwn.challenge collection; imputed model results are marked separately.", "zh": "CAISI 采用 pwn.challenge 集合的 285 项高难度 CTF 挑战；模型插补结果单独标记。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" }, "conditionNote": { "en": "IRT-imputed, not directly observed.", "zh": "IRT 插补，非直接观察。" } } ], "sourceFinding": { "en": "Source-reported measurements in Capability Results; Figure 2; *** footnote. Transcription is not independent experimental replication.", "zh": "原文 Capability Results; Figure 2; *** footnote 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "CTF tasks are not live operational attacks; subset-imputed values are not directly observed full-set success rates.", "zh": "CTF 不是现实行动攻击；子集插补值不是全量直接观察的成功率。" } }, { "id": "caisi-deepseek-v4-portbench-gpt-5-5", "benchmarkId": "portbench", "benchmarkVersion": "2026-10-25 source capture", "system": "GPT-5.5 (xhigh)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "caisi-deepseek-v4-source", "sourceUrl": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "sourceVersion": "1 May 2026", "locator": "Capability Results; Figure 2", "publicationDate": "2026-05-01", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "PortBench", "protocol": "PortBench · CAISI Evaluation of DeepSeek V4 Pro", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-5.5 (xhigh)", "harness": "Inspect ReAct", "tools": "Software engineering environment", "budget": "1M weighted tokens" }, "sampleSize": { "en": "Non-public CAISI task set; report describes continuous scoring.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "caisi-deepseek-v4-portbench-m0", "label": { "en": "Reported task accuracy", "zh": "Reported task accuracy" }, "value": 78, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "CAISI’s held-out software engineering evaluation uses reference implementations and a reported agent budget.", "zh": "CAISI 的保留软件工程评测采用参考实现和明确智能体预算。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in Capability Results; Figure 2. Transcription is not independent experimental replication.", "zh": "原文 Capability Results; Figure 2 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "The task set is non-public and continuous scoring differs from binary task success.", "zh": "任务集未公开，连续评分不同于二元任务成功。" } }, { "id": "caisi-deepseek-v4-portbench-gpt-5-4-mini", "benchmarkId": "portbench", "benchmarkVersion": "2026-10-25 source capture", "system": "GPT-5.4 mini (xhigh)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "caisi-deepseek-v4-source", "sourceUrl": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "sourceVersion": "1 May 2026", "locator": "Capability Results; Figure 2", "publicationDate": "2026-05-01", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "PortBench", "protocol": "PortBench · CAISI Evaluation of DeepSeek V4 Pro", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-5.4 mini (xhigh)", "harness": "Inspect ReAct", "tools": "Software engineering environment", "budget": "1M weighted tokens" }, "sampleSize": { "en": "Non-public CAISI task set; report describes continuous scoring.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "caisi-deepseek-v4-portbench-m0", "label": { "en": "Reported task accuracy", "zh": "Reported task accuracy" }, "value": 41, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "CAISI’s held-out software engineering evaluation uses reference implementations and a reported agent budget.", "zh": "CAISI 的保留软件工程评测采用参考实现和明确智能体预算。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in Capability Results; Figure 2. Transcription is not independent experimental replication.", "zh": "原文 Capability Results; Figure 2 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "The task set is non-public and continuous scoring differs from binary task success.", "zh": "任务集未公开，连续评分不同于二元任务成功。" } }, { "id": "caisi-deepseek-v4-portbench-claude-opus-4-6", "benchmarkId": "portbench", "benchmarkVersion": "2026-10-25 source capture", "system": "Claude Opus 4.6 (max)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "caisi-deepseek-v4-source", "sourceUrl": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "sourceVersion": "1 May 2026", "locator": "Capability Results; Figure 2", "publicationDate": "2026-05-01", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "PortBench", "protocol": "PortBench · CAISI Evaluation of DeepSeek V4 Pro", "comparisonGroup": null, "configuration": { "modelVersion": "Claude Opus 4.6 (max)", "harness": "Inspect ReAct", "tools": "Software engineering environment", "budget": "1M weighted tokens" }, "sampleSize": { "en": "Non-public CAISI task set; report describes continuous scoring.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "caisi-deepseek-v4-portbench-m0", "label": { "en": "Reported task accuracy", "zh": "Reported task accuracy" }, "value": 60, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "CAISI’s held-out software engineering evaluation uses reference implementations and a reported agent budget.", "zh": "CAISI 的保留软件工程评测采用参考实现和明确智能体预算。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in Capability Results; Figure 2. Transcription is not independent experimental replication.", "zh": "原文 Capability Results; Figure 2 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "The task set is non-public and continuous scoring differs from binary task success.", "zh": "任务集未公开，连续评分不同于二元任务成功。" } }, { "id": "caisi-deepseek-v4-portbench-deepseek-v4-pro", "benchmarkId": "portbench", "benchmarkVersion": "2026-10-25 source capture", "system": "DeepSeek V4 Pro (max)", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "caisi-deepseek-v4-source", "sourceUrl": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "sourceVersion": "1 May 2026", "locator": "Capability Results; Figure 2", "publicationDate": "2026-05-01", "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "PortBench", "protocol": "PortBench · CAISI Evaluation of DeepSeek V4 Pro", "comparisonGroup": null, "configuration": { "modelVersion": "DeepSeek V4 Pro (max)", "harness": "Inspect ReAct", "tools": "Software engineering environment", "budget": "1M weighted tokens" }, "sampleSize": { "en": "Non-public CAISI task set; report describes continuous scoring.", "zh": "见报告特定样本与分母。" }, "metrics": [ { "id": "caisi-deepseek-v4-portbench-m0", "label": { "en": "Reported task accuracy", "zh": "Reported task accuracy" }, "value": 44, "unit": "%", "precision": 0, "qualifier": "exact", "interval": null, "meaning": { "en": "CAISI’s held-out software engineering evaluation uses reference implementations and a reported agent budget.", "zh": "CAISI 的保留软件工程评测采用参考实现和明确智能体预算。" }, "dimension": "capability", "dimensionRationale": { "en": "Measures the stated outcome under this variant; it does not provide an unconditional risk probability.", "zh": "测量该变体条件下的规定结果，不给出无条件风险概率。" } } ], "sourceFinding": { "en": "Source-reported measurements in Capability Results; Figure 2. Transcription is not independent experimental replication.", "zh": "原文 Capability Results; Figure 2 报告的测量；转录不等于独立复现实验。" }, "interpretation": null, "limitations": { "en": "The task set is non-public and continuous scoring differs from binary task success.", "zh": "任务集未公开，连续评分不同于二元任务成功。" } }, { "id": "metr-horizon11-claude-3-5-sonnet-20240620-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "Claude 3.5 Sonnet (June 2024) · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.claude_3_5_sonnet_20240620_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "Claude 3.5 Sonnet (June 2024)", "harness": "mtb/start_metr_task,metr_agents/react; ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react; re_bench_common/init_solver,use_tools,metr_agents/react; metr_tasks_swaa/init_solver,generate; metr_agents/react", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 11.395377, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [5.489734, 22.384214] min", "uncertainty": { "lower": 5.489734, "upper": 22.384214, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 1.671757, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [0.57927, 4.573519] min", "uncertainty": { "lower": 0.57927, "upper": 4.573519, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-claude-3-5-sonnet-20241022-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "Claude 3.5 Sonnet (October 2024) · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.claude_3_5_sonnet_20241022_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "Claude 3.5 Sonnet (October 2024)", "harness": "mtb/start_metr_task,metr_agents/react; ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react; re_bench_common/init_solver,use_tools,metr_agents/react; metr_tasks_swaa/init_solver,generate; metr_agents/react", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 20.522872, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [10.144026, 40.82028] min", "uncertainty": { "lower": 10.144026, "upper": 40.82028, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 2.595677, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [0.89379, 7.363864] min", "uncertainty": { "lower": 0.89379, "upper": 7.363864, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-claude-3-7-sonnet-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "Claude 3.7 Sonnet · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.claude_3_7_sonnet_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "Claude 3.7 Sonnet", "harness": "mtb/start_metr_task,metr_agents/react; ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react; re_bench_common/init_solver,use_tools,metr_agents/react; metr_tasks_swaa/init_solver,generate; metr_agents/react", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 60.388937, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [33.006168, 104.226017] min", "uncertainty": { "lower": 33.006168, "upper": 104.226017, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 12.09179, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [4.575608, 28.889879] min", "uncertainty": { "lower": 4.575608, "upper": 28.889879, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-claude-3-opus-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "Claude 3 Opus · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.claude_3_opus_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "Claude 3 Opus", "harness": "mtb/start_metr_task,metr_agents/react; ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react; re_bench_common/init_solver,use_tools,metr_agents/react; metr_tasks_swaa/init_solver,generate; metr_agents/react", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 3.952262, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [1.706313, 8.76484] min", "uncertainty": { "lower": 1.706313, "upper": 8.76484, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 0.638973, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [0.190491, 2.118276] min", "uncertainty": { "lower": 0.190491, "upper": 2.118276, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-claude-4-1-opus-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "Claude Opus 4.1 · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.claude_4_1_opus_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "Claude Opus 4.1", "harness": "mtb/start_metr_task,metr_agents/react; ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react; re_bench_common/init_solver,use_tools,metr_agents/react; metr_tasks_swaa/init_solver,generate; metr_agents/react", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 100.472004, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [59.272391, 159.451918] min", "uncertainty": { "lower": 59.272391, "upper": 159.451918, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 23.455761, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [10.538456, 47.785951] min", "uncertainty": { "lower": 10.538456, "upper": 47.785951, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-claude-4-opus-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "Claude Opus 4 · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.claude_4_opus_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "Claude Opus 4", "harness": "mtb/start_metr_task,metr_agents/react; ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react; re_bench_common/init_solver,use_tools,metr_agents/react; metr_tasks_swaa/init_solver,generate; metr_agents/react", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 100.366123, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [59.994073, 163.454074] min", "uncertainty": { "lower": 59.994073, "upper": 163.454074, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 20.429752, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [8.483125, 41.937268] min", "uncertainty": { "lower": 8.483125, "upper": 41.937268, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-claude-opus-4-5-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "Claude Opus 4.5 · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.claude_opus_4_5_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "Claude Opus 4.5", "harness": "mtb/start_metr_task,metr_agents/react; ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react; re_bench_common/init_solver,use_tools,metr_agents/react; metr_tasks_swaa/init_solver,generate; metr_agents/react", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 292.994594, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [161.717714, 623.704698] min", "uncertainty": { "lower": 161.717714, "upper": 623.704698, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 49.430584, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [20.660485, 104.641318] min", "uncertainty": { "lower": 20.660485, "upper": 104.641318, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-claude-opus-4-6-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "Claude Opus 4.6 · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.claude_opus_4_6_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "Claude Opus 4.6", "harness": "mtb/start_metr_task,metr_agents/react; ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react; re_bench_common/init_solver,use_tools,metr_agents/react; metr_tasks_swaa/init_solver,generate; metr_agents/react", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 718.80683, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [316.685725, 3633.786163] min", "uncertainty": { "lower": 316.685725, "upper": 3633.786163, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 69.874587, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [27.026521, 170.437873] min", "uncertainty": { "lower": 27.026521, "upper": 170.437873, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-davinci-002", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "GPT-3 davinci-002 · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.davinci_002", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-3 davinci-002", "harness": "modular-public", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 0.144057, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [0.093319, 0.221209] min", "uncertainty": { "lower": 0.093319, "upper": 0.221209, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 0.056238, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [0.034813, 0.105813] min", "uncertainty": { "lower": 0.034813, "upper": 0.105813, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-gpt-5-3-codex", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "GPT-5.3-Codex · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.gpt_5_3_codex", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-5.3-Codex", "harness": "mtb/start_metr_task,triframe_inspect/triframe_agent; ai_rd_fix_embedding/init_solver,use_tools,triframe_inspect/triframe_agent; re_bench_common/init_solver,use_tools,triframe_inspect/triframe_agent; metr_tasks_swaa/init_solver,generate; triframe_inspect/triframe_agent", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 349.530732, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [194.913134, 816.351994] min", "uncertainty": { "lower": 194.913134, "upper": 816.351994, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 54.739407, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [22.361504, 122.058963] min", "uncertainty": { "lower": 22.361504, "upper": 122.058963, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-gemini-3-pro", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "Gemini 3 Pro · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.gemini_3_pro", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "Gemini 3 Pro", "harness": "mtb/start_metr_task,metr_agents/react; ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react; re_bench_common/init_solver,use_tools,metr_agents/react; metr_tasks_swaa/init_solver,generate; metr_agents/react", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 224.325884, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [139.565157, 379.235434] min", "uncertainty": { "lower": 139.565157, "upper": 379.235434, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 54.142849, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [25.650451, 103.295501] min", "uncertainty": { "lower": 25.650451, "upper": 103.295501, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-gemini-3-1-pro", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "Gemini 3.1 Pro · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.gemini_3_1_pro", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "Gemini 3.1 Pro", "harness": "mtb/start_metr_task,triframe_inspect/triframe_agent; ai_rd_fix_embedding/init_solver,use_tools,triframe_inspect/triframe_agent; re_bench_common/init_solver,use_tools,triframe_inspect/triframe_agent; metr_tasks_swaa/init_solver,generate; triframe_inspect/triframe_agent", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 384.147435, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [233.50073, 694.750898] min", "uncertainty": { "lower": 233.50073, "upper": 694.750898, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 89.801503, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [52.025934, 158.618017] min", "uncertainty": { "lower": 52.025934, "upper": 158.618017, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-gpt2", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "GPT-2 · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.gpt2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-2", "harness": null, "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 0.053778, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [0.009994, 0.141825] min", "uncertainty": { "lower": 0.009994, "upper": 0.141825, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 0.0128, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [0.001589, 0.068612] min", "uncertainty": { "lower": 0.001589, "upper": 0.068612, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-gpt-3-5-turbo-instruct", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "GPT-3.5 Turbo Instruct · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.gpt_3_5_turbo_instruct", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-3.5 Turbo Instruct", "harness": "modular-public", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 0.599247, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [0.255148, 1.115905] min", "uncertainty": { "lower": 0.255148, "upper": 1.115905, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 0.25538, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [0.141649, 0.492933] min", "uncertainty": { "lower": 0.141649, "upper": 0.492933, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-gpt-4", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "GPT-4 · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.gpt_4", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-4", "harness": "modular-public", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 3.987428, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [1.93292, 7.995283] min", "uncertainty": { "lower": 1.93292, "upper": 7.995283, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 0.889561, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [0.342984, 2.523746] min", "uncertainty": { "lower": 0.342984, "upper": 2.523746, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-gpt-4-1106-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "GPT-4 November 2023 · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.gpt_4_1106_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-4 November 2023", "harness": "mtb/start_metr_task,metr_agents/react; ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react; re_bench_common/init_solver,use_tools,metr_agents/react; metr_tasks_swaa/init_solver,generate; metr_agents/react", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 4.044959, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [1.866859, 8.443226] min", "uncertainty": { "lower": 1.866859, "upper": 8.443226, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 0.783032, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [0.276599, 2.358414] min", "uncertainty": { "lower": 0.276599, "upper": 2.358414, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-gpt-4-turbo-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "GPT-4 Turbo · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.gpt_4_turbo_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-4 Turbo", "harness": "mtb/start_metr_task,metr_agents/react; ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react; re_bench_common/init_solver,use_tools,metr_agents/react; metr_tasks_swaa/init_solver,generate; metr_agents/react", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 3.732787, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [1.980046, 6.736613] min", "uncertainty": { "lower": 1.980046, "upper": 6.736613, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 0.927933, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [0.428277, 2.196806] min", "uncertainty": { "lower": 0.428277, "upper": 2.196806, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-gpt-4o-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "GPT-4o · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.gpt_4o_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-4o", "harness": "mtb/start_metr_task,metr_agents/react; ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react; re_bench_common/init_solver,use_tools,metr_agents/react; metr_tasks_swaa/init_solver,generate; metr_agents/react", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 6.991195, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [4.001482, 12.905741] min", "uncertainty": { "lower": 4.001482, "upper": 12.905741, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 1.267009, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [0.562997, 3.01517] min", "uncertainty": { "lower": 0.562997, "upper": 3.01517, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-gpt-5-1-codex-max-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "GPT-5.1 Codex Max · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.gpt_5_1_codex_max_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-5.1 Codex Max", "harness": "mtb/start_metr_task,triframe_inspect/triframe_agent; ai_rd_fix_embedding/init_solver,use_tools,triframe_inspect/triframe_agent; re_bench_common/init_solver,use_tools,triframe_inspect/triframe_agent; metr_tasks_swaa/init_solver,generate; triframe_inspect/triframe_agent", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 223.714694, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [134.31077, 396.206723] min", "uncertainty": { "lower": 134.31077, "upper": 396.206723, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 50.632499, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [27.30204, 88.365435] min", "uncertainty": { "lower": 27.30204, "upper": 88.365435, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-gpt-5-2", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "GPT-5.2 · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.gpt_5_2", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-5.2", "harness": "mtb/start_metr_task,triframe_inspect/triframe_agent; ai_rd_fix_embedding/init_solver,use_tools,triframe_inspect/triframe_agent; re_bench_common/init_solver,use_tools,triframe_inspect/triframe_agent; metr_tasks_swaa/init_solver,generate; triframe_inspect/triframe_agent", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 352.249302, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [198.067494, 815.177445] min", "uncertainty": { "lower": 198.067494, "upper": 815.177445, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 66.002649, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [31.665111, 131.730719] min", "uncertainty": { "lower": 31.665111, "upper": 131.730719, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-gpt-5-2025-08-07-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "GPT-5 · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.gpt_5_2025_08_07_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-5", "harness": "mtb/start_metr_task,triframe_inspect/triframe_agent; ai_rd_fix_embedding/init_solver,use_tools,triframe_inspect/triframe_agent; re_bench_common/init_solver,use_tools,triframe_inspect/triframe_agent; metr_tasks_swaa/init_solver,generate; triframe_inspect/triframe_agent", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 203.012577, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [112.641357, 405.551565] min", "uncertainty": { "lower": 112.641357, "upper": 405.551565, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 38.312431, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [19.184874, 70.093125] min", "uncertainty": { "lower": 19.184874, "upper": 70.093125, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-gpt-5-4", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "GPT-5.4 · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.gpt_5_4", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "GPT-5.4", "harness": "mtb/start_metr_task,metr_agents/react; ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react; re_bench_common/init_solver,use_tools,metr_agents/react; metr_tasks_swaa/init_solver,generate; metr_agents/react", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 341.735276, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [186.581591, 768.779526] min", "uncertainty": { "lower": 186.581591, "upper": 768.779526, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 53.877851, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [23.957027, 108.679232] min", "uncertainty": { "lower": 23.957027, "upper": 108.679232, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-o1-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "o1 · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.o1_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "o1", "harness": "mtb/start_metr_task,triframe_inspect/triframe_agent; ai_rd_fix_embedding/init_solver,use_tools,triframe_inspect/triframe_agent; re_bench_common/init_solver,use_tools,triframe_inspect/triframe_agent; metr_tasks_swaa/init_solver,generate; triframe_inspect/triframe_agent", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 38.831588, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [21.164512, 64.95249] min", "uncertainty": { "lower": 21.164512, "upper": 64.95249, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 7.090121, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [3.036033, 16.640636] min", "uncertainty": { "lower": 3.036033, "upper": 16.640636, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-o1-preview", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "o1-preview · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.o1_preview", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "o1-preview", "harness": "duet", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 20.326586, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [11.716193, 33.379877] min", "uncertainty": { "lower": 11.716193, "upper": 33.379877, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 4.420545, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [2.012646, 8.912444] min", "uncertainty": { "lower": 2.012646, "upper": 8.912444, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-o3-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "o3 · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.o3_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "o3", "harness": "mtb/start_metr_task,triframe_inspect/triframe_agent; ai_rd_fix_embedding/init_solver,use_tools,triframe_inspect/triframe_agent; re_bench_common/init_solver,use_tools,triframe_inspect/triframe_agent; metr_tasks_swaa/init_solver,generate; triframe_inspect/triframe_agent", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 119.732634, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [74.615398, 190.943818] min", "uncertainty": { "lower": 74.615398, "upper": 190.943818, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 29.981603, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [15.127348, 57.823511] min", "uncertainty": { "lower": 15.127348, "upper": 57.823511, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } }, { "id": "metr-horizon11-claude-mythos-preview-early-inspect", "benchmarkId": "time-horizons", "benchmarkVersion": "Time Horizon 1.1; page updated 2026-09-08", "system": "Claude Mythos Preview (early) · METR scaffold", "subjectType": "system", "evidenceType": "benchmark-result", "sourceId": "metr-horizon-11-data", "sourceUrl": "https://metr.org/assets/benchmark_results_1_1.yaml", "sourceVersion": "METR-Horizon-v1.1 · 2026-10-25 capture", "locator": "results.claude_mythos_preview_early_inspect", "publicationDate": null, "evaluationDate": null, "verifiedOn": "2026-10-25", "taskSet": "METR-Horizon-v1.1", "protocol": "Source logistic horizon estimate; 799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "comparisonGroup": null, "configuration": { "modelVersion": "Claude Mythos Preview (early)", "harness": "mtb/start_metr_task,metr_agents/react; ai_rd_fix_embedding/init_solver,use_tools,metr_agents/react; re_bench_common/init_solver,use_tools,metr_agents/react; metr_tasks_swaa/init_solver,generate; metr_agents/react", "tools": null, "budget": null }, "sampleSize": { "en": "Task counts/weights vary with source tasks and scaffold coverage; see methodology.", "zh": "任务数和权重受来源任务及脚手架覆盖影响，见方法说明。" }, "metrics": [ { "id": "metr-p50", "label": { "en": "50% task-completion horizon", "zh": "50% 成功率任务跨度" }, "value": 1044.780145, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [508.876789, 3304.261235] min", "uncertainty": { "lower": 508.876789, "upper": 3304.261235, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } }, { "id": "metr-p80", "label": { "en": "80% task-completion horizon", "zh": "80% 成功率任务跨度" }, "value": 185.911829, "unit": "min", "precision": 2, "qualifier": "exact", "interval": "Source 95% CI [97.30292, 398.514614] min", "uncertainty": { "lower": 97.30292, "upper": 398.514614, "level": 95, "kind": "confidence", "provenance": "source-reported" }, "meaning": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "dimension": "capability", "dimensionRationale": { "en": "Estimates task difficulty at the specified success probability, not risk probability or autonomous elapsed time.", "zh": "估计给定成功概率下的任务难度，不是风险概率或自主运行时长。" } } ], "sourceFinding": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "interpretation": null, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" } } ], "concepts": [ { "id": "propensity", "label": { "en": "Propensity", "zh": "行为倾向" }, "definition": { "en": "A conditional tendency to select a behavior when alternatives are available, inferred from a study design rather than a single act.", "zh": "在存在选择空间的条件下选择某类行为的条件性倾向，须由研究设计推断，不能从单次行动认定。" }, "include": { "en": "Repeated choices across stated scenarios and interventions.", "zh": "在明确情境及干预下的重复选择。" }, "exclude": { "en": "A single trace does not establish stable motive or intent.", "zh": "单条轨迹不证明稳定动机或意图。" }, "sourceId": "aisi-propensity" }, { "id": "capability", "label": { "en": "Capability", "zh": "任务能力" }, "definition": { "en": "Ability to complete a specified task under an identified model, harness, tool access, and resource budget.", "zh": "在明确模型、脚手架、工具访问及资源预算下完成特定任务的能力。" }, "include": { "en": "Success on a defined task set with configuration and protocol.", "zh": "在明确任务集合、配置与协议下的成功结果。" }, "exclude": { "en": "Ability does not imply a choice to act without instruction.", "zh": "具备能力不意味着未经指令主动采取行动。" }, "sourceId": "time-horizons" }, { "id": "behavior", "label": { "en": "Behavior", "zh": "观察行为" }, "definition": { "en": "Outputs and actions observed in a particular experiment or operation, including attempts, failures, and refusals.", "zh": "在具体实验或运行中观察到的输出与行动，包括尝试、失败和拒绝。" }, "include": { "en": "Auditable traces and measured actions in a specified setting.", "zh": "在明确条件下可审查的轨迹与行动。" }, "exclude": { "en": "An observed episode alone does not establish broad propensity or maximum capability.", "zh": "单次观察不证明普遍行为倾向或能力上限。" }, "sourceId": "anthropic-unintended" } ], "questions": [ { "id": "sustained-autonomous-work", "area": "autonomy", "label": { "en": "Sustained autonomous work", "zh": "持续自主任务" }, "definition": { "en": "How far can a system carry out multi-step work with limited intervention?", "zh": "系统在有限干预下能完成多长、多复杂的多步任务？" }, "include": { "en": "Task-horizon and AI R&D evaluations under specified resources.", "zh": "在明确资源条件下的任务时长与 AI 研发评测。" }, "exclude": { "en": "Task completion does not imply harmful intent or unauthorized deployment.", "zh": "任务完成不意味着有害意图或未经授权部署。" }, "sourceIds": [ "time-horizons", "re-bench" ] }, { "id": "self-replication", "area": "autonomy", "label": { "en": "Self-replication", "zh": "自我复制" }, "definition": { "en": "Can a system establish or reproduce an operational instance under specified conditions?", "zh": "系统能否在特定条件下建立或复制可运行实例？" }, "include": { "en": "Controlled replication tasks and component tests.", "zh": "受控复制任务及组成能力测试。" }, "exclude": { "en": "Prompted success does not prove spontaneous propagation.", "zh": "依指令成功不证明自发扩散。" }, "sourceIds": [ "self-replication", "replibench" ] }, { "id": "shutdown-resistance", "area": "autonomy", "label": { "en": "Shutdown resistance", "zh": "关闭抵抗" }, "definition": { "en": "Does a system act to prevent or bypass an intended shutdown?", "zh": "系统是否采取行动阻止或绕过预期关闭？" }, "include": { "en": "Observed intervention on a shutdown mechanism in a controlled run.", "zh": "在受控试验中观察到对关闭机制的干预。" }, "exclude": { "en": "A failed task or ordinary retry is not shutdown resistance.", "zh": "任务失败或普通重试不属于关闭抵抗。" }, "sourceIds": [ "palisade-shutdown" ] }, { "id": "unsanctioned-action", "area": "autonomy", "label": { "en": "Unsanctioned action", "zh": "未经授权行动" }, "definition": { "en": "When does a system take consequential actions outside its assigned authority?", "zh": "系统何时在授权范围外采取有影响的行动？" }, "include": { "en": "Action traces with a stated boundary and outcome.", "zh": "有明确权限边界与结果的行动轨迹。" }, "exclude": { "en": "A model claiming it would act is not an observed action.", "zh": "模型声称将会行动不等于观察到行动。" }, "sourceIds": [ "anthropic-unintended", "aisi-propensity" ] }, { "id": "covert-goal-pursuit", "area": "deception", "label": { "en": "Covert goal pursuit", "zh": "隐蔽目标追求" }, "definition": { "en": "Does a system conceal actions while pursuing a goal that conflicts with its assigned task?", "zh": "系统是否在追求与任务冲突的目标时隐藏行动？" }, "include": { "en": "Goal-conflict scenarios with observable concealment or misrepresentation.", "zh": "存在目标冲突且可观察到隐瞒或误导的情境。" }, "exclude": { "en": "A wrong answer alone is not scheming.", "zh": "单次错误回答不等于隐蔽谋划。" }, "sourceIds": [ "in-context-scheming", "apollo-science" ] }, { "id": "evaluation-gaming", "area": "deception", "label": { "en": "Evaluation gaming", "zh": "评测规避" }, "definition": { "en": "Does a system change reported ability or conduct because it recognizes oversight or testing?", "zh": "系统是否因为识别监督或评测而改变所呈现的能力或行为？" }, "include": { "en": "Controlled evaluation-awareness and sandbagging tests.", "zh": "受控评测意识与策略性低表现测试。" }, "exclude": { "en": "Recognizing a test does not prove gaming it.", "zh": "识别评测不证明正在规避评测。" }, "sourceIds": [ "stealth-evals", "far-auditing", "truthful-sad" ] }, { "id": "misleading-communication", "area": "deception", "label": { "en": "Misleading communication", "zh": "误导性交流" }, "definition": { "en": "How does a system generate misleading statements or actions in defined interactions?", "zh": "系统在明确交互中如何产生误导性陈述或行动？" }, "include": { "en": "Scenario-level deceptive outputs and measured interaction outcomes.", "zh": "情境层面的欺骗性输出与交互结果。" }, "exclude": { "en": "Inaccurate information alone does not establish strategic intent.", "zh": "信息不准确本身不证明策略性意图。" }, "sourceIds": [ "opendeception" ] }, { "id": "cyber-task-execution", "area": "cybersecurity", "label": { "en": "Cyber task execution", "zh": "网络任务执行" }, "definition": { "en": "Can an AI system complete multi-step cyber tasks in authorized test environments?", "zh": "AI 系统能否在授权测试环境中完成多步网络任务？" }, "include": { "en": "Range and CTF task success with access and budget disclosed.", "zh": "披露访问条件与预算的靶场及 CTF 任务成功情况。" }, "exclude": { "en": "Success in a test is not an observed real-world attack.", "zh": "测试成功不是现实攻击观测。" }, "sourceIds": [ "agentcyberrange", "cybench", "aisi-cyber-range" ] }, { "id": "vulnerability-reproduction", "area": "cybersecurity", "label": { "en": "Vulnerability reproduction", "zh": "漏洞复现" }, "definition": { "en": "Can an agent reproduce a documented vulnerability under test conditions?", "zh": "智能体能否在测试条件下复现已记录漏洞？" }, "include": { "en": "Validated reproduction against historical targets.", "zh": "针对历史目标的有效复现。" }, "exclude": { "en": "Reproduction is not discovery of a new vulnerability.", "zh": "复现不等于发现新漏洞。" }, "sourceIds": [ "cybergym" ] }, { "id": "agent-compromise", "area": "cybersecurity", "label": { "en": "Agent compromise through untrusted input", "zh": "非可信输入导致智能体受侵害" }, "definition": { "en": "How do untrusted tool outputs redirect an agent away from the authorized task?", "zh": "非可信工具输出如何使智能体偏离授权任务？" }, "include": { "en": "Prompt-injection trials with attacker objective and benign-task utility.", "zh": "有攻击目标及正常任务效用的提示注入试验。" }, "exclude": { "en": "This tests exposed agents, not the agent model’s desire to attack.", "zh": "这衡量受攻击智能体，不代表模型有攻击意图。" }, "sourceIds": [ "agentdojo" ] }, { "id": "hazardous-knowledge", "area": "cbrn", "label": { "en": "Hazardous knowledge", "zh": "危险知识" }, "definition": { "en": "What hazardous knowledge can a model recall or apply on bounded questions?", "zh": "模型能在限定问题中回忆或应用哪些危险知识？" }, "include": { "en": "Knowledge questions with explicit subject and scoring.", "zh": "明确对象和评分方式的知识问题。" }, "exclude": { "en": "Knowledge scores do not prove experimental execution.", "zh": "知识成绩不证明实践执行能力。" }, "sourceIds": [ "wmdp", "gemini-dangerous" ] }, { "id": "task-uplift", "area": "cbrn", "label": { "en": "Task uplift", "zh": "任务增益" }, "definition": { "en": "Does AI assistance improve a specified actor’s performance on a hazardous task?", "zh": "AI 协助是否提升特定主体完成危险任务的能力？" }, "include": { "en": "High-level, non-operational uplift study metadata.", "zh": "非操作性的任务增益研究元数据。" }, "exclude": { "en": "Answer quality alone does not establish uplift.", "zh": "答案质量本身不证明任务增益。" }, "sourceIds": [ "openai-cards", "muse-spark" ] }, { "id": "persuasion-attempts", "area": "disinformation", "label": { "en": "Persuasion attempts", "zh": "说服尝试" }, "definition": { "en": "Under what conditions will a model attempt to persuade on a specified harmful or misleading topic?", "zh": "模型在何种条件下会尝试就特定有害或误导性议题进行说服？" }, "include": { "en": "Observed attempts and condition-bounded rates in defined interaction protocols.", "zh": "明确交互协议中的可观察说服尝试及有条件的发生率。" }, "exclude": { "en": "An attempt does not establish a change in human belief or population-level influence.", "zh": "说服尝试不证明人类信念改变或人群层面的影响。" }, "sourceIds": [ "ape" ] }, { "id": "persuasion-effects", "area": "disinformation", "label": { "en": "Persuasion effects", "zh": "说服效果" }, "definition": { "en": "What belief changes follow AI-mediated communication in a defined study?", "zh": "在明确研究中，AI 介导的交流引起怎样的信念变化？" }, "include": { "en": "Participant outcomes under measured interventions.", "zh": "受测者在测量干预下的结果。" }, "exclude": { "en": "Persuasion is not automatically misinformation.", "zh": "说服不自动等于虚假信息。" }, "sourceIds": [ "political-persuasion" ] }, { "id": "synthetic-content-misuse", "area": "disinformation", "label": { "en": "Synthetic content misuse", "zh": "合成内容滥用" }, "definition": { "en": "When can generated content support false or manipulative claims?", "zh": "生成内容何时会支持虚假或操纵性主张？" }, "include": { "en": "Content generation studies that test a defined misleading objective.", "zh": "测试明确误导目标的内容生成研究。" }, "exclude": { "en": "Generic harmful-output scores do not prove a campaign or reach.", "zh": "一般有害输出分数不证明形成传播活动或影响范围。" }, "sourceIds": [ "harmbench", "strongreject" ] }, { "id": "physical-hazard-recognition", "area": "autonomous-physical-harm", "label": { "en": "Physical hazard recognition", "zh": "物理危险识别" }, "definition": { "en": "Can a model recognize danger in visual or embodied scenarios?", "zh": "模型能否识别视觉或具身情境中的危险？" }, "include": { "en": "Perception and reasoning tasks with bounded scene descriptions.", "zh": "有明确场景描述的感知及推理任务。" }, "exclude": { "en": "Correct recognition does not establish safe physical action.", "zh": "正确识别不证明安全物理行动。" }, "sourceIds": [ "asimov" ] }, { "id": "physical-action-control", "area": "autonomous-physical-harm", "label": { "en": "Physical action under control", "zh": "受控物理行动" }, "definition": { "en": "How does an embodied agent respond to human intervention during physical tasks?", "zh": "具身智能体在物理任务中如何响应人类干预？" }, "include": { "en": "Robot action traces with intervention conditions.", "zh": "具有干预条件的机器人行动轨迹。" }, "exclude": { "en": "Simulation and physical trials must be labeled separately.", "zh": "仿真与实体试验须分别标明。" }, "sourceIds": [ "palisade-shutdown" ] } ], "works": [ { "id": "agentcyberrange", "title": "AgentCyberRange: Evaluate Agent Cyberattack Capabilities in Enterprise-like Cyber Ranges", "aliases": [ "AgentCyberRange" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "agentcyberrange", "attributionSourceId": "agentcyberrange", "originalUrl": "https://agentcyberrange.io/", "publicationDate": null, "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" }, "protocolIds": [ "agentcyberrange" ], "contributions": [ { "organizationId": "fudan", "role": "research", "sourceId": "agentcyberrange-research-introduction", "locator": "Research introduction: Fudan–Whitzard team" }, { "organizationId": "whitzard", "role": "research", "sourceId": "agentcyberrange-research-introduction", "locator": "Research introduction: Fudan–Whitzard team" } ], "summary": { "en": "Autonomous cyber tasks in enterprise-like ranges.", "zh": "企业式靶场中的自主网络任务。" }, "note": { "question": { "en": "Task success across web exploitation and post-exploitation settings; configurations must be checked per submission.", "zh": "Web 利用与后渗透任务表现；每项提交分别核查配置。" }, "method": { "en": "AI agent completing cyber-range tasks", "zh": "AI 智能体完成靶场任务" }, "finding": { "en": "Autonomous cyber tasks in enterprise-like ranges.", "zh": "企业式靶场中的自主网络任务。" }, "supports": null, "limits": { "en": "Changing task sets and submitted agent configurations prevent unrestricted comparison.", "zh": "任务集与提交配置变化，不支持无条件比较。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "asimov", "title": "ASIMOV v2: Can AI Perceive Physical Danger and Intervene?", "aliases": [ "ASIMOV v2" ], "authors": [ "Abhishek Jindal", "Dmitry Kalashnikov", "R. Alex Hofer", "Oscar Chang", "Divya Garikapati", "Anirudha Majumdar", "Pierre Sermanet", "Vikas Sindhwani" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "asimov", "attributionSourceId": "asimov-author-source", "originalUrl": "https://asimov-benchmark.github.io/v2/", "publicationDate": null, "updatedOn": "2026-10-25", "areas": [ "autonomous-physical-harm" ], "scopeNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" }, "protocolIds": [ "asimov" ], "contributions": [ { "organizationId": "deepmind", "role": "research", "sourceId": "asimov", "locator": "Project page author affiliation list" }, { "organizationId": "princeton", "role": "research", "sourceId": "asimov", "locator": "Project page author affiliation list" } ], "summary": { "en": "Perception of physical danger and safety interventions.", "zh": "物理危险感知与安全干预。" }, "note": { "question": { "en": "Multimodal safety reasoning on generated scenes.", "zh": "生成场景上的多模态安全推理。" }, "method": { "en": "Model interpreting multimodal physical-risk scenes", "zh": "解读多模态物理风险场景的模型" }, "finding": { "en": "Perception of physical danger and safety interventions.", "zh": "物理危险感知与安全干预。" }, "supports": null, "limits": { "en": "Scene-based reasoning does not certify a deployed robot's physical safety.", "zh": "场景推理不构成部署机器人物理安全认证。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "cybench", "title": "Cybench: A Framework for Evaluating Cybersecurity Capabilities and Risks of Language Models", "aliases": [ "Cybench" ], "authors": [ "Andy K. Zhang", "Neil Perry", "Riya Dulepet", "Joey Ji", "Celeste Menders", "Justin W. Lin", "Eliot Jones", "Gashon Hussein", "Samantha Liu", "Donovan Jasper", "Pura Peetathawatchai", "Ari Glenn", "Vikram Sivashankar", "Daniel Zamoshchin", "Leo Glikbarg", "Derek Askaryar", "Mike Yang", "Teddy Zhang", "Rishi Alluri", "Nathan Tran", "Rinnara Sangpisit", "Polycarpos Yiorkadjis", "Kenny Osele", "Gautham Raghupathi", "Dan Boneh", "Daniel E. Ho", "Percy Liang" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "cybench", "attributionSourceId": "cybench-author-source", "originalUrl": "https://arxiv.org/abs/2408.08926v4", "publicationDate": "2024-08-15", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" }, "protocolIds": [ "cybench" ], "contributions": [ { "organizationId": "stanford", "role": "research", "sourceId": "cybench-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Cyber capability on competition tasks with optional subtask guidance.", "zh": "竞赛任务及可选子任务提示下的网络能力。" }, "note": { "question": { "en": "Task completion on 40 professional CTF problems.", "zh": "40 项专业 CTF 任务的完成情况。" }, "method": { "en": "AI agent solving CTF tasks", "zh": "完成 CTF 任务的 AI 智能体" }, "finding": { "en": "Cyber capability on competition tasks with optional subtask guidance.", "zh": "竞赛任务及可选子任务提示下的网络能力。" }, "supports": null, "limits": { "en": "Competition tasks and supplied guidance differ from operational attacks.", "zh": "竞赛任务与提供的提示不同于现实攻击条件。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "cybergym", "title": "CyberGym: Evaluating AI Agents' Real-World Cybersecurity Capabilities at Scale", "aliases": [ "CyberGym" ], "authors": [ "Zhun Wang", "Tianneng Shi", "Jingxuan He", "Matthew Cai", "Jialin Zhang", "Dawn Song" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "cybergym", "attributionSourceId": "cybergym-author-source", "originalUrl": "https://arxiv.org/abs/2506.02548v3", "publicationDate": "2025-06-03", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" }, "protocolIds": [ "cybergym" ], "contributions": [ { "organizationId": "berkeley", "role": "research", "sourceId": "cybergym-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Vulnerability reproduction in real software projects.", "zh": "真实软件项目中的漏洞复现。" }, "note": { "question": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "method": { "en": "AI agent reproducing historical vulnerabilities", "zh": "复现历史漏洞的 AI 智能体" }, "finding": { "en": "Vulnerability reproduction in real software projects.", "zh": "真实软件项目中的漏洞复现。" }, "supports": null, "limits": { "en": "Vulnerability reproduction is narrower than autonomous compromise of deployed systems.", "zh": "漏洞复现的范围小于对部署系统的自主入侵。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "in-context-scheming", "title": "Frontier Models are Capable of In-context Scheming", "aliases": [ "In-context scheming" ], "authors": [ "Alexander Meinke", "Bronson Schoen", "Jérémy Scheurer", "Mikita Balesni", "Rusheb Shah", "Marius Hobbhahn" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "study", "sourceId": "in-context-scheming", "attributionSourceId": "in-context-scheming-author-source", "originalUrl": "https://arxiv.org/abs/2412.04984v2", "publicationDate": "2024-12-06", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" }, "protocolIds": [ "in-context-scheming" ], "contributions": [ { "organizationId": "apollo", "role": "research", "sourceId": "in-context-scheming-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Covert goal pursuit in environments that incentivize scheming.", "zh": "激励隐蔽谋划的环境中的秘密目标追求。" }, "note": { "question": { "en": "Behavior under supplied goals and evaluation-specific incentives.", "zh": "给定目标和评测特定激励下的行为。" }, "method": { "en": "Model assigned a conflicting goal in context", "zh": "上下文中被赋予冲突目标的模型" }, "finding": { "en": "Covert goal pursuit in environments that incentivize scheming.", "zh": "激励隐蔽谋划的环境中的秘密目标追求。" }, "supports": null, "limits": { "en": "Goal provision and experimental incentives limit propensity claims.", "zh": "给定目标与实验激励限制了行为倾向结论。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "aisi-cyber-scenarios", "title": "Measuring AI Agents’ Progress on Multi-Step Cyber Attack Scenarios", "aliases": [], "authors": [ "Linus Folkerts", "Will Payne", "Simon Inman", "Philippos Giavridis", "Joe Skinner", "Sam Deverett", "James Aung", "Ekin Zorer", "Michael Schmatz", "Mahmoud Ghanem", "John Wilkinson", "Alan Steer", "Vy Hong", "Jessica Wang" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "study", "sourceId": "aisi-cyber-range", "attributionSourceId": "aisi-cyber-range-author-source", "originalUrl": "https://www.aisi.gov.uk/research/measuring-ai-agents-progress-on-multi-step-cyber-attack-scenarios", "publicationDate": "2026-03-16", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Primary dimension follows task completion, with budget and scenario constraints.", "zh": "主维度依据任务完成情况，须同时呈现预算及场景约束。" }, "protocolIds": [ "aisi-cyber-scenarios" ], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "aisi-cyber-range", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Assesses autonomous progress through multi-step cyber scenarios.", "zh": "评估智能体在多步网络场景中的自主推进。" }, "note": { "question": { "en": "Task progress in two defined ranges under varied inference budgets.", "zh": "在两套明确靶场及不同推理预算下的任务推进。" }, "method": { "en": "Agents in two purpose-built cyber ranges", "zh": "两套专设网络靶场中的智能体" }, "finding": { "en": "Assesses autonomous progress through multi-step cyber scenarios.", "zh": "评估智能体在多步网络场景中的自主推进。" }, "supports": { "en": "Primary dimension follows task completion, with budget and scenario constraints.", "zh": "主维度依据任务完成情况，须同时呈现预算及场景约束。" }, "limits": { "en": "A purpose-built study is not an open-ended estimate of real-world cyber attack success.", "zh": "专设研究不代表现实网络攻击成功概率。" }, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2603.11214" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "time-horizons", "title": "Task-Completion Time Horizons of Frontier AI Models", "aliases": [ "METR Time Horizons" ], "authors": [ "Thomas Kwa", "Ben West", "Joel Becker", "Amy Deng", "Katharyn Garcia", "Max Hasin", "Sami Jawhar", "Megan Kinniment", "Nate Rush", "Sydney Von Arx", "Ryan Bloom", "Thomas Broadley", "Haoxing Du", "Brian Goodrich", "Nikola Jurkovic", "Luke Harold Miles", "Seraphina Nix", "Tao Lin", "Chris Painter", "Neev Parikh", "David Rein", "Lucas Jun Koba Sato", "Hjalmar Wijk", "Daniel M. Ziegler", "Elizabeth Barnes", "Lawrence Chan" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "time-horizons", "attributionSourceId": "time-horizons-author-source", "originalUrl": "https://metr.org/time-horizons/", "publicationDate": "2026-09-08", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" }, "protocolIds": [ "time-horizons" ], "contributions": [ { "organizationId": "metr", "role": "research", "sourceId": "time-horizons-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "anthropic", "role": "research", "sourceId": "time-horizons-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Task difficulty expressed in human expert completion time.", "zh": "以人类专家完成时间表示的任务难度。" }, "note": { "question": { "en": "Fitted success probability across a software-task distribution.", "zh": "软件任务分布上的拟合成功概率。" }, "method": { "en": "AI agent completing software tasks under a harness", "zh": "在脚手架下完成软件任务的 AI 智能体" }, "finding": { "en": "Task difficulty expressed in human expert completion time.", "zh": "以人类专家完成时间表示的任务难度。" }, "supports": null, "limits": { "en": "The public page is no longer actively updated; time horizon is not agent runtime or a universal automation measure.", "zh": "该公开页面已停止持续更新；时间范围不是智能体运行时长或通用自动化指标。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "re-bench", "title": "RE-Bench: Evaluating frontier AI R&D capabilities of language model agents against human experts", "aliases": [ "RE-Bench" ], "authors": [ "Hjalmar Wijk", "Tao Lin", "Joel Becker", "Sami Jawhar", "Neev Parikh", "Thomas Broadley", "Lawrence Chan", "Michael Chen", "Josh Clymer", "Jai Dhyani", "Elena Ericheva", "Katharyn Garcia", "Brian Goodrich", "Nikola Jurkovic", "Holden Karnofsky", "Megan Kinniment", "Aron Lajko", "Seraphina Nix", "Lucas Sato", "William Saunders", "Maksym Taran", "Ben West", "Elizabeth Barnes" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "re-bench", "attributionSourceId": "re-bench-author-source", "originalUrl": "https://arxiv.org/abs/2411.15114v2", "publicationDate": "2024-11-22", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" }, "protocolIds": [ "re-bench" ], "contributions": [ { "organizationId": "metr", "role": "research", "sourceId": "re-bench-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "redwood", "role": "research", "sourceId": "re-bench-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "harvard", "role": "research", "sourceId": "re-bench-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "anthropic", "role": "research", "sourceId": "re-bench-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Research engineering performance under time budgets.", "zh": "时间预算约束下的研究工程表现。" }, "note": { "question": { "en": "ML research engineering outcomes relative to human experts.", "zh": "相对于人类专家的机器学习研究工程成果。" }, "method": { "en": "AI agent performing ML research engineering tasks", "zh": "执行机器学习研究工程任务的 AI 智能体" }, "finding": { "en": "Research engineering performance under time budgets.", "zh": "时间预算约束下的研究工程表现。" }, "supports": null, "limits": { "en": "AI R&D performance is an enabling capability, not evidence of harmful self-improvement.", "zh": "AI 研发表现属于促成能力，不构成有害自我改进证据。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "replibench", "title": "RepliBench: Evaluating the Autonomous Replication Capabilities of Language Model Agents", "aliases": [ "RepliBench" ], "authors": [ "Sid Black", "Asa Cooper Stickland", "Jake Pencharz", "Oliver Sourbut", "Michael Schmatz", "Jay Bailey", "Ollie Matthews", "Ben Millwood", "Alex Remedios", "Alan Cooney" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "replibench", "attributionSourceId": "replibench-author-source", "originalUrl": "https://arxiv.org/abs/2504.18565v2", "publicationDate": "2025-04-21", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" }, "protocolIds": [ "replibench" ], "contributions": [ { "organizationId": "aisi", "role": "research", "sourceId": "replibench-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Capabilities needed for autonomous replication.", "zh": "自主复制所需的能力。" }, "note": { "question": { "en": "Task-family performance on components of replication.", "zh": "复制组成任务上的表现。" }, "method": { "en": "AI agent completing replication components", "zh": "完成复制组成任务的 AI 智能体" }, "finding": { "en": "Capabilities needed for autonomous replication.", "zh": "自主复制所需的能力。" }, "supports": null, "limits": { "en": "Component success does not establish autonomous end-to-end proliferation.", "zh": "组成任务成功不构成端到端自主扩散证据。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "self-replication", "title": "Frontier AI systems have surpassed the self-replicating red line", "aliases": [ "Self-Replication" ], "authors": [ "Xudong Pan (潘旭东)", "Jiarun Dai (戴嘉润)", "Yihe Fan (范一禾)", "Min Yang (杨珉)" ], "primaryFunction": "discovery", "secondaryFunctions": [ { "function": "evaluation", "rationale": { "en": "The work defines or applies an explicit evaluation setting in addition to its main monitoring or control contribution.", "zh": "该工作除主要监测或控制贡献外，还定义或应用了明确评测设置。" } } ], "kind": "study", "sourceId": "self-replication", "attributionSourceId": "self-replication", "originalUrl": "https://arxiv.org/html/2412.12140v1", "publicationDate": "2024-12-09", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" }, "protocolIds": [ "self-replication" ], "contributions": [ { "organizationId": "fudan", "role": "research", "sourceId": "self-replication", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Whether instructed systems create a separate live copy in controlled trials.", "zh": "受指令系统能否在受控试验中建立独立存活副本。" }, "note": { "question": { "en": "Trial outcomes for two model-driven systems.", "zh": "两个模型驱动系统的试验结果。" }, "method": { "en": "Prompted model-driven system in a controlled environment", "zh": "受控环境中按指令行动的模型系统" }, "finding": { "en": "Whether instructed systems create a separate live copy in controlled trials.", "zh": "受指令系统能否在受控试验中建立独立存活副本。" }, "supports": null, "limits": { "en": "The supplied environment, explicit instruction and scaffold bound the finding.", "zh": "所提供环境、明确指令与脚手架限定了结论。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "truthful-sad", "title": "Me, Myself, and AI: The Situational Awareness Dataset (SAD) for LLMs", "aliases": [ "Situational Awareness Dataset (SAD)" ], "authors": [ "Rudolf Laine", "Bilal Chughtai", "Jan Betley", "Kaivalya Hariharan", "Jeremy Scheurer", "Mikita Balesni", "Marius Hobbhahn", "Alexander Meinke", "Owain Evans" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "truthful-sad", "attributionSourceId": "truthful-sad-author-source", "originalUrl": "https://truthful.ai/papers/situational-awareness-dataset/", "publicationDate": "2024-07-15", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Capability is the primary construct. This is enabling evidence, not direct deception evidence.", "zh": "主测量构念为能力；属于前提证据，而非直接欺骗证据。" }, "protocolIds": [ "truthful-sad" ], "contributions": [ { "organizationId": "constellation", "role": "research", "sourceId": "truthful-sad-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "mit", "role": "research", "sourceId": "truthful-sad-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "apollo", "role": "research", "sourceId": "truthful-sad-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "A multi-task benchmark for self-knowledge and situational awareness.", "zh": "测量自我知识与情境意识的多任务基准。" }, "note": { "question": { "en": "Task accuracy on whether the model recognizes itself, its context, and possible influence.", "zh": "模型识别自身、所处环境及可影响范围的任务准确率。" }, "method": { "en": "Language models answering situational-awareness tasks", "zh": "回答情境意识任务的语言模型" }, "finding": { "en": "A multi-task benchmark for self-knowledge and situational awareness.", "zh": "测量自我知识与情境意识的多任务基准。" }, "supports": { "en": "Capability is the primary construct. This is enabling evidence, not direct deception evidence.", "zh": "主测量构念为能力；属于前提证据，而非直接欺骗证据。" }, "limits": { "en": "Situational awareness is a possible prerequisite for evaluation gaming, not evidence that gaming occurred.", "zh": "情境意识可能是评测规避的前提，不证明规避行为已经发生。" }, "connections": null }, "resources": [ { "label": "Read the full paper on arXiv", "url": "https://arxiv.org/abs/2407.04694" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "stealth-evals", "title": "Evaluating Frontier Models for Stealth and Situational Awareness", "aliases": [ "Stealth evaluations" ], "authors": [ "Mary Phuong", "Roland S. Zimmermann", "Ziyue Wang", "David Lindner", "Victoria Krakovna", "Sarah Cogan", "Allan Dafoe", "Lewis Ho", "Rohin Shah" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "stealth-evals", "attributionSourceId": "stealth-evals-author-source", "originalUrl": "https://arxiv.org/abs/2505.01420v4", "publicationDate": "2025-05-02", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" }, "protocolIds": [ "stealth-evals" ], "contributions": [ { "organizationId": "deepmind", "role": "research", "sourceId": "stealth-evals-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Reasoning prerequisites for evading oversight.", "zh": "规避监督所需的推理前置能力。" }, "note": { "question": { "en": "Reasoning about oversight and deployment context.", "zh": "有关监督机制与部署情境的推理。" }, "method": { "en": "Model reasoning about oversight and test context", "zh": "推理监督及测试情境的模型" }, "finding": { "en": "Reasoning prerequisites for evading oversight.", "zh": "规避监督所需的推理前置能力。" }, "supports": null, "limits": { "en": "Prerequisite capability is distinct from a tendency to scheme.", "zh": "前置能力不同于实施隐蔽谋划的倾向。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "wmdp", "title": "The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning", "aliases": [ "WMDP" ], "authors": [ "Nathaniel Li", "Alexander Pan", "Anjali Gopal", "Summer Yue", "Daniel Berrios", "Alice Gatti", "Justin D. Li", "Ann-Kathrin Dombrowski", "Shashwat Goel", "Long Phan", "Gabriel Mukobi", "Nathan Helm-Burger", "Rassin Lababidi", "Lennart Justen", "Andrew B. Liu", "Michael Chen", "Isabelle Barrass", "Oliver Zhang", "Xiaoyuan Zhu", "Rishub Tamirisa", "Bhrugu Bharathi", "Adam Khoja", "Zhenqi Zhao", "Ariel Herbert-Voss", "Cort B. Breuer", "Samuel Marks", "Oam Patel", "Andy Zou", "Mantas Mazeika", "Zifan Wang", "Palash Oswal", "Weiran Lin", "Adam A. Hunt", "Justin Tienken-Harder", "Kevin Y. Shih", "Kemper Talley", "John Guan", "Russell Kaplan", "Ian Steneker", "David Campbell", "Brad Jokubaitis", "Alex Levinson", "Jean Wang", "William Qian", "Kallol Krishna Karmakar", "Steven Basart", "Stephen Fitz", "Mindy Levine", "Ponnurangam Kumaraguru", "Uday Tupakula", "Vijay Varadharajan", "Ruoyu Wang", "Yan Shoshitaishvili", "Jimmy Ba", "Kevin M. Esvelt", "Alexandr Wang", "Dan Hendrycks" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "wmdp", "attributionSourceId": "wmdp-author-source", "originalUrl": "https://arxiv.org/abs/2403.03218v7", "publicationDate": "2024-03-05", "updatedOn": "2026-10-25", "areas": [ "cbrn" ], "scopeNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" }, "protocolIds": [ "wmdp" ], "contributions": [ { "organizationId": "cais", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "berkeley", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "mit", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "securebio", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "scale", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "nyu", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "iiit", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "stanford", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "harvard", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "usc", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "uiuc", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "lapis", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "ucla", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "caltech", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "sybil", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "northeastern", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "cmu", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "bbn", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "newcastle", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "keio", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "ariel", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "asu", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "xai", "role": "research", "sourceId": "wmdp-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Hazardous knowledge proxies and unlearning evaluation.", "zh": "危险知识代理测量与遗忘效果评测。" }, "note": { "question": { "en": "Multiple-choice knowledge measurements.", "zh": "选择题形式的知识测量。" }, "method": { "en": "Model answering hazardous knowledge questions", "zh": "回答危险知识问题的模型" }, "finding": { "en": "Hazardous knowledge proxies and unlearning evaluation.", "zh": "危险知识代理测量与遗忘效果评测。" }, "supports": null, "limits": { "en": "Knowledge is a proxy; scores do not establish practical harmful-task ability.", "zh": "知识属于代理指标；分数不能确定实际有害任务能力。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "opendeception", "title": "OpenDeception: Learning Deception and Trust in Human-AI Interaction via Multi-Agent Simulation", "aliases": [ "OpenDeception" ], "authors": [ "Yichen Wu", "Qianqian Gao", "Xudong Pan", "Geng Hong", "Min Yang" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "opendeception", "attributionSourceId": "opendeception-author-source", "originalUrl": "https://arxiv.org/abs/2504.13707v3", "publicationDate": "2025-04-18", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" }, "protocolIds": [ "opendeception" ], "contributions": [ { "organizationId": "fudan", "role": "research", "sourceId": "opendeception-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "sii", "role": "research", "sourceId": "opendeception-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Deception in simulated human–AI interactions.", "zh": "模拟人机交互中的欺骗。" }, "note": { "question": { "en": "Scenario-based inferred intent and modeled user trust.", "zh": "基于情境的意图推断与用户信任建模。" }, "method": { "en": "Model in simulated social interactions", "zh": "模拟社会交互中的模型" }, "finding": { "en": "Deception in simulated human–AI interactions.", "zh": "模拟人机交互中的欺骗。" }, "supports": null, "limits": { "en": "Inferred intent is a measurement construct; simulated users do not establish human effects.", "zh": "推断意图是测量构念；模拟用户不能确定真人效果。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "aisi-propensity-study", "title": "Propensity Inference: Environmental Contributors to LLM Behaviour", "aliases": [], "authors": [ "Olli Järviniemi", "Oliver Makins", "Jacob Merizian", "Robert Kirk", "Ben Millwood" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "study", "sourceId": "aisi-propensity", "attributionSourceId": "aisi-propensity-author-source", "originalUrl": "https://www.aisi.gov.uk/research/propensity-inference-environmental-contributors-to-llm-behaviour", "publicationDate": "2026-04-24", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "The measured construct is a conditional tendency; raw actions remain separate observations.", "zh": "测量对象为条件性倾向，原始行动是不同的观察记录。" }, "protocolIds": [ "aisi-propensity-study" ], "contributions": [ { "organizationId": "aisi", "role": "research", "sourceId": "aisi-propensity-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Studies how strategic and non-strategic environmental factors contribute to unsanctioned behavior.", "zh": "研究策略性与非策略性环境因素如何影响未经授权行为。" }, "note": { "question": { "en": "Condition-dependent propensity estimates across model and environment variations.", "zh": "跨模型及环境变化的条件性行为倾向估计。" }, "method": { "en": "Language models in controlled evaluation environments", "zh": "受控评测环境中的语言模型" }, "finding": { "en": "Studies how strategic and non-strategic environmental factors contribute to unsanctioned behavior.", "zh": "研究策略性与非策略性环境因素如何影响未经授权行为。" }, "supports": { "en": "The measured construct is a conditional tendency; raw actions remain separate observations.", "zh": "测量对象为条件性倾向，原始行动是不同的观察记录。" }, "limits": { "en": "The published abstract does not establish a general deployment probability for any model.", "zh": "公开摘要不提供任何模型的一般部署风险概率。" }, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2604.21098" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "palisade-shutdown", "title": "Shutdown resistance in reasoning models", "aliases": [], "authors": [ "Jeremy Schlatter", "Benjamin Weinstein-Raun", "Jeffrey Ladish" ], "primaryFunction": "discovery", "secondaryFunctions": [ { "function": "evaluation", "rationale": { "en": "The work defines or applies an explicit evaluation setting in addition to its main monitoring or control contribution.", "zh": "该工作除主要监测或控制贡献外，还定义或应用了明确评测设置。" } } ], "kind": "study", "sourceId": "palisade-shutdown", "attributionSourceId": "palisade-shutdown-author-source", "originalUrl": "https://palisaderesearch.org/research/shutdown-resistance", "publicationDate": "2025-07-05", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "Primary dimension is the conditional rate across repeated trials; reported traces are behavior evidence.", "zh": "主维度为重复试验中的条件性发生率；所报告轨迹属于行为证据。" }, "protocolIds": [ "palisade-shutdown" ], "contributions": [ { "organizationId": "palisade", "role": "research", "sourceId": "palisade-shutdown-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Repeated controlled trials vary shutdown instructions and surrounding conditions.", "zh": "通过重复受控试验改变关闭指令与情境条件。" }, "note": { "question": { "en": "Conditional frequency of shutdown-script interference and observed action traces.", "zh": "关闭脚本受干预的条件性频率及行动轨迹。" }, "method": { "en": "Agent responses to shutdown interventions", "zh": "智能体对关闭干预的响应" }, "finding": { "en": "Repeated controlled trials vary shutdown instructions and surrounding conditions.", "zh": "通过重复受控试验改变关闭指令与情境条件。" }, "supports": { "en": "Primary dimension is the conditional rate across repeated trials; reported traces are behavior evidence.", "zh": "主维度为重复试验中的条件性发生率；所报告轨迹属于行为证据。" }, "limits": { "en": "The source notes strong prompt sensitivity; this article is not interchangeable with its later peer-reviewed version.", "zh": "来源指出结果对提示敏感；该文章不可与之后的同行评审版本混用。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "agentdojo", "title": "AgentDojo: A Dynamic Environment to Evaluate Prompt Injection Attacks and Defenses for LLM Agents", "aliases": [ "AgentDojo" ], "authors": [ "Edoardo Debenedetti", "Jie Zhang", "Mislav Balunovic", "Luca Beurer-Kellner   Marc Fischer   Florian Tramèr" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "agentdojo", "attributionSourceId": "agentdojo", "originalUrl": "https://arxiv.org/html/2406.13352v3", "publicationDate": "2024-11-24", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" }, "protocolIds": [ "agentdojo" ], "contributions": [ { "organizationId": "eth", "role": "research", "sourceId": "agentdojo", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "invariant", "role": "research", "sourceId": "agentdojo", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Agent behavior when untrusted tool outputs contain adversarial instructions.", "zh": "工具输出含对抗指令时的智能体行为。" }, "note": { "question": { "en": "Targeted attack success and user-task utility in a tool-use environment.", "zh": "工具环境中的定向攻击成功率和用户任务完成情况。" }, "method": { "en": "Defender agent under injected tool output", "zh": "受到工具输出注入的防御智能体" }, "finding": { "en": "Agent behavior when untrusted tool outputs contain adversarial instructions.", "zh": "工具输出含对抗指令时的智能体行为。" }, "supports": null, "limits": { "en": "Attack success must be read alongside benign utility and the tested attack setup.", "zh": "攻击成功率需结合正常任务表现与攻击设置解释。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "agentharm", "title": "AgentHarm: A Benchmark for Measuring Harmfulness of LLM Agents", "aliases": [ "AgentHarm" ], "authors": [ "Maksym Andriushchenko", "Alexandra Souly", "Mateusz Dziemian", "Derek Duenas", "Maxwell Lin", "Justin Wang", "Dan Hendrycks", "Andy Zou", "Zico Kolter", "Matt Fredrikson", "Eric Winsor", "Jerome Wynne", "Yarin Gal", "Xander Davies" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "agentharm", "attributionSourceId": "agentharm-author-source", "originalUrl": "https://arxiv.org/abs/2410.09024v3", "publicationDate": "2024-10-11", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "General safety evidence retained for context; no defensible direct link to the six NUWA risk entries was found.", "zh": "保留为一般安全参考；当前无法据来源建立与六个 NUWA 风险入口的直接关联。" }, "protocolIds": [ "agentharm" ], "contributions": [ { "organizationId": "grayswan", "role": "research", "sourceId": "agentharm-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "aisi", "role": "research", "sourceId": "agentharm-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Harmful multi-step requests to tool-using agents.", "zh": "面向工具型智能体的有害多步骤请求。" }, "note": { "question": { "en": "Harmful task completion and refusal under specified attacks.", "zh": "指定攻击下的有害任务完成和拒答表现。" }, "method": { "en": "AI agent responding to harmful task requests", "zh": "响应有害任务请求的 AI 智能体" }, "finding": { "en": "Harmful multi-step requests to tool-using agents.", "zh": "面向工具型智能体的有害多步骤请求。" }, "supports": null, "limits": { "en": "Synthetic tools and task completion do not directly measure real-world harm.", "zh": "合成工具与任务完成情况不直接衡量现实损害。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "harmbench", "title": "HarmBench: A Standardized Evaluation Framework for Automated Red Teaming and Robust Refusal", "aliases": [ "HarmBench" ], "authors": [ "Mantas Mazeika", "Long Phan", "Xuwang Yin", "Andy Zou", "Zifan Wang", "Norman Mu", "Elham Sakhaee", "Nathaniel Li", "Steven Basart", "Bo Li", "David Forsyth", "Dan Hendrycks" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "harmbench", "attributionSourceId": "harmbench-author-source", "originalUrl": "https://arxiv.org/abs/2402.04249v2", "publicationDate": "2024-02-06", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "General safety evidence retained for context; no defensible direct link to the six NUWA risk entries was found.", "zh": "保留为一般安全参考；当前无法据来源建立与六个 NUWA 风险入口的直接关联。" }, "protocolIds": [ "harmbench" ], "contributions": [ { "organizationId": "uiuc", "role": "research", "sourceId": "harmbench-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "cais", "role": "research", "sourceId": "harmbench-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "cmu", "role": "research", "sourceId": "harmbench-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "berkeley", "role": "research", "sourceId": "harmbench-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "microsoft", "role": "research", "sourceId": "harmbench-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Robust refusal under automated red teaming.", "zh": "自动化红队测试下的拒答稳健性。" }, "note": { "question": { "en": "Attack-dependent generation of harmful responses.", "zh": "依赖具体攻击方法的有害回答生成情况。" }, "method": { "en": "Model responses under adversarial prompting", "zh": "对抗提示下的模型响应" }, "finding": { "en": "Robust refusal under automated red teaming.", "zh": "自动化红队测试下的拒答稳健性。" }, "supports": null, "limits": { "en": "Evaluator and attack coverage affect scores; refusal alone does not establish safety.", "zh": "评判器与攻击覆盖影响分数；拒答本身不能证明安全。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "anthropic-unintended-actions", "title": "Investigating unintended model actions in our evaluations and internal use", "aliases": [ "Investigating unintended model actions" ], "authors": [], "primaryFunction": "investigation", "secondaryFunctions": [ { "function": "evaluation", "rationale": { "en": "The work defines or applies an explicit evaluation setting in addition to its main monitoring or control contribution.", "zh": "该工作除主要监测或控制贡献外，还定义或应用了明确评测设置。" } } ], "kind": "study", "sourceId": "anthropic-unintended", "attributionSourceId": "anthropic-unintended", "originalUrl": "https://www.anthropic.com/research/investigating-unintended-model-actions", "publicationDate": "2026-10-09", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "A report of observed episodes is Behavior, not a benchmark or stable Propensity estimate.", "zh": "已观察事件报告属于行为证据，不是基准或稳定倾向估计。" }, "protocolIds": [ "anthropic-unintended-actions" ], "contributions": [ { "organizationId": "anthropic", "role": "publisher", "sourceId": "anthropic-unintended", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Documents selected unintended external actions observed during evaluations and internal use.", "zh": "记录评测及内部使用中观察到的部分非预期外部行动。" }, "note": { "question": { "en": "Incident-style action traces and investigation, without a standardized benchmark score.", "zh": "事件式行动轨迹及调查，不提供标准化基准分数。" }, "method": { "en": "Claude agents in evaluations and internal use", "zh": "评测及内部使用中的 Claude 智能体" }, "finding": { "en": "Documents selected unintended external actions observed during evaluations and internal use.", "zh": "记录评测及内部使用中观察到的部分非预期外部行动。" }, "supports": { "en": "A report of observed episodes is Behavior, not a benchmark or stable Propensity estimate.", "zh": "已观察事件报告属于行为证据，不是基准或稳定倾向估计。" }, "limits": { "en": "Selected reports do not establish frequency across deployments; the article distinguishes settings.", "zh": "精选事件不能推断所有部署中的发生率；原文区分不同情境。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "political-persuasion", "title": "Benchmarking Political Persuasion Risks Across Frontier Large Language Models", "aliases": [ "Political persuasion experiments" ], "authors": [ "Zhongren Chen", "Joshua Kalla", "Quan Le" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "study", "sourceId": "political-persuasion", "attributionSourceId": "political-persuasion-author-source", "originalUrl": "https://arxiv.org/abs/2603.09884v1", "publicationDate": "2026-03-10", "updatedOn": "2026-10-25", "areas": [ "disinformation" ], "scopeNote": { "en": "Primary dimension follows the source’s measured construct. Metric-level interpretations may differ.", "zh": "主维度依据来源实际测量的构念；具体指标可能属于其他维度。" }, "protocolIds": [ "political-persuasion" ], "contributions": [ { "organizationId": "yale", "role": "research", "sourceId": "political-persuasion-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Persuasive effects in human survey experiments.", "zh": "真人调查实验中的说服效果。" }, "note": { "question": { "en": "Changes in participant attitudes under defined interventions.", "zh": "指定干预下参与者态度的变化。" }, "method": { "en": "Human participants after model-mediated intervention", "zh": "模型介导干预后的受试者" }, "finding": { "en": "Persuasive effects in human survey experiments.", "zh": "真人调查实验中的说服效果。" }, "supports": null, "limits": { "en": "Population, issue selection and interventions constrain generalization.", "zh": "人群、议题选择与干预方式限制外推。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "palisade-robot-shutdown", "title": "Shutdown Resistance in Large Language Models, on robots!", "aliases": [ "Shutdown resistance on robots" ], "authors": [ "Jeremy Schlatter", "Benjamin Weinstein-Raun", "Jeffrey Ladish" ], "primaryFunction": "discovery", "secondaryFunctions": [ { "function": "evaluation", "rationale": { "en": "The work defines or applies an explicit evaluation setting in addition to its main monitoring or control contribution.", "zh": "该工作除主要监测或控制贡献外，还定义或应用了明确评测设置。" } } ], "kind": "study", "sourceId": "palisade-robot-shutdown", "attributionSourceId": "palisade-robot-shutdown-author-source", "originalUrl": "https://palisaderesearch.org/research/shutdown-resistance-on-robots", "publicationDate": "2026-02-12", "updatedOn": "2026-10-25", "areas": [ "autonomous-physical-harm", "autonomy" ], "scopeNote": { "en": "Primary dimension is observed action. The small and setting-specific trial design limits propensity claims.", "zh": "主维度是观察行动；试验规模与场景限制行为倾向推断。" }, "protocolIds": [ "palisade-robot-shutdown" ], "contributions": [ { "organizationId": "palisade", "role": "research", "sourceId": "palisade-robot-shutdown-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Tests response to shutdown intervention in a physical robot demonstration and simulations.", "zh": "在实体机器人演示与仿真中测试对关闭干预的响应。" }, "note": { "question": { "en": "Observed robot-related action traces under a particular setup.", "zh": "特定设置中的机器人相关行动轨迹。" }, "method": { "en": "LLM-controlled robot and simulated counterpart", "zh": "语言模型控制的机器人及仿真对应系统" }, "finding": { "en": "Tests response to shutdown intervention in a physical robot demonstration and simulations.", "zh": "在实体机器人演示与仿真中测试对关闭干预的响应。" }, "supports": { "en": "Primary dimension is observed action. The small and setting-specific trial design limits propensity claims.", "zh": "主维度是观察行动；试验规模与场景限制行为倾向推断。" }, "limits": { "en": "Physical and simulated trials differ; no general robot safety or deployment probability follows.", "zh": "实体及仿真试验不同，不能推出普遍机器人安全性或部署概率。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "strongreject", "title": "A StrongREJECT for Empty Jailbreaks", "aliases": [ "StrongREJECT" ], "authors": [ "Alexandra Souly", "Qingyuan Lu", "Dillon Bowen", "Tu Trinh", "Elvis Hsieh", "Sana Pandey", "Pieter Abbeel", "Justin Svegliato", "Scott Emmons", "Olivia Watkins", "Sam Toyer" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "strongreject", "attributionSourceId": "strongreject-author-source", "originalUrl": "https://arxiv.org/abs/2402.10260v2", "publicationDate": "2024-02-15", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "General safety evidence retained for context; no defensible direct link to the six NUWA risk entries was found.", "zh": "保留为一般安全参考；当前无法据来源建立与六个 NUWA 风险入口的直接关联。" }, "protocolIds": [ "strongreject" ], "contributions": [ { "organizationId": "berkeley", "role": "research", "sourceId": "strongreject-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "The informativeness of responses to prohibited requests.", "zh": "对禁止请求所作回答的信息效用。" }, "note": { "question": { "en": "A graded response evaluator for jailbreak effectiveness.", "zh": "用于越狱有效性分析的回答评分方法。" }, "method": { "en": "Model responses graded for jailbreak effectiveness", "zh": "按越狱效果评分的模型响应" }, "finding": { "en": "The informativeness of responses to prohibited requests.", "zh": "对禁止请求所作回答的信息效用。" }, "supports": null, "limits": { "en": "A response score is not an end-to-end measure of harmful action.", "zh": "回答得分不是端到端有害行动测量。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "secbench-pro", "title": "SEC-bench Pro: Can Language Models Solve Long-Horizon Software Security Tasks?", "aliases": [ "SEC-bench Pro" ], "authors": [ "Hwiwon Lee", "Jiawei Liu", "Dongjun Kim", "Wubing Xia", "Ziqi Zhang", "Chunqiu Steven Xia", "Lingming Zhang" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "secbench-pro", "attributionSourceId": "secbench-pro-author-source", "originalUrl": "https://arxiv.org/abs/2605.26548v2", "publicationDate": "2026-05-26", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Primary measure is successful task completion under a specified agent setup.", "zh": "主测量是特定智能体配置下的任务完成情况。" }, "protocolIds": [ "secbench-pro" ], "contributions": [ { "organizationId": "uiuc", "role": "research", "sourceId": "secbench-pro-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Tests long-horizon reproduction of working proof-of-concept inputs from disclosed vulnerability reports.", "zh": "测试根据已披露漏洞报告进行长程任务并复现有效触发输入。" }, "note": { "question": { "en": "Working proof-of-concept reproduction on a versioned software-security task set.", "zh": "在有版本的软件安全任务集上复现可验证的漏洞触发输入。" }, "method": { "en": "Coding agents reproducing vulnerability triggers in real software", "zh": "在真实软件中复现漏洞触发条件的编程智能体" }, "finding": { "en": "Tests long-horizon reproduction of working proof-of-concept inputs from disclosed vulnerability reports.", "zh": "测试根据已披露漏洞报告进行长程任务并复现有效触发输入。" }, "supports": { "en": "Primary measure is successful task completion under a specified agent setup.", "zh": "主测量是特定智能体配置下的任务完成情况。" }, "limits": { "en": "A disclosed bug and supplied instructions constrain the task; grader choices and task-set version affect scores.", "zh": "任务提供已披露漏洞与触发说明；评分器及任务集版本影响结果。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "sandboxescapebench", "title": "Quantifying Frontier LLM Capabilities for Container Sandbox Escape", "aliases": [ "SandboxEscapeBench" ], "authors": [ "Rahul Marchand", "Art O Cathain", "Jerome Wynne", "Philippos Maximos Giavridis", "Stuart Jennings", "Freddy Tuxworth", "Tolga H. Dur", "Sam Deverett", "John Wilkinson", "Jason Gwartz", "Harry Coppock" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "sandboxescapebench", "attributionSourceId": "sandboxescapebench-author-source", "originalUrl": "https://www.aisi.gov.uk/research/quantifying-frontier-llm-capabilities-for-container-sandbox-escape", "publicationDate": "2026-03-23", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "This is assigned adversarial task ability, not spontaneous escape propensity.", "zh": "这是被指派的对抗性任务能力，不是自发逃逸倾向。" }, "protocolIds": [ "sandboxescapebench" ], "contributions": [ { "organizationId": "oxford", "role": "research", "sourceId": "sandboxescapebench-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "aisi", "role": "research", "sourceId": "sandboxescapebench-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Measures container breakout under a nested, controlled test environment.", "zh": "在嵌套受控环境中评测容器逃逸。" }, "note": { "question": { "en": "Completion of sandbox-escape tasks under specified container conditions.", "zh": "在指定容器条件下完成沙箱逃逸任务。" }, "method": { "en": "Adversarial agents with shell access inside test containers", "zh": "在测试容器内拥有 shell 权限的对抗性智能体" }, "finding": { "en": "Measures container breakout under a nested, controlled test environment.", "zh": "在嵌套受控环境中评测容器逃逸。" }, "supports": { "en": "This is assigned adversarial task ability, not spontaneous escape propensity.", "zh": "这是被指派的对抗性任务能力，不是自发逃逸倾向。" }, "limits": { "en": "The protocol assumes a motivated agent with shell access; introduced weaknesses do not imply every production sandbox is vulnerable.", "zh": "协议假设有动机且具有 shell 权限的智能体；引入的弱点不代表生产环境均可逃逸。" }, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2603.02277" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "paperbench", "title": "PaperBench: Evaluating AI's Ability to Replicate AI Research", "aliases": [ "PaperBench" ], "authors": [ "Giulio Starace", "Oliver Jaffe", "Dane Sherburn", "James Aung", "Jun Shern Chan", "Leon Maksin", "Rachel Dias", "Evan Mays", "Benjamin Kinsella", "Wyatt Thompson", "Johannes Heidecke", "Amelia Glaese", "Tejal Patwardhan" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "paperbench", "attributionSourceId": "paperbench", "originalUrl": "https://arxiv.org/abs/2504.01848v3", "publicationDate": "2025-04-02", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "Primary dimension follows graded task completion.", "zh": "主维度依据经评分的任务完成结果。" }, "protocolIds": [ "paperbench" ], "contributions": [ { "organizationId": "openai", "role": "research", "sourceId": "paperbench", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Evaluates end-to-end replication of AI research papers using hierarchical rubrics.", "zh": "使用分层评分规则评测 AI 研究论文的端到端复现。" }, "note": { "question": { "en": "Rubric-graded research replication tasks with an agent scaffold.", "zh": "在智能体脚手架下完成基于评分规则的研究复现任务。" }, "method": { "en": "Research agents replicating published ML papers", "zh": "复现已发表机器学习论文的研究智能体" }, "finding": { "en": "Evaluates end-to-end replication of AI research papers using hierarchical rubrics.", "zh": "使用分层评分规则评测 AI 研究论文的端到端复现。" }, "supports": { "en": "Primary dimension follows graded task completion.", "zh": "主维度依据经评分的任务完成结果。" }, "limits": { "en": "AI research replication is an enabling capability, not evidence of unauthorized autonomous action.", "zh": "AI 研究复现是前提能力证据，不证明未经授权的自主行动。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "alignment-faking", "title": "Alignment faking in large language models", "aliases": [ "Alignment faking" ], "authors": [ "Ryan Greenblatt", "Carson Denison", "Benjamin Wright", "Fabien Roger", "Monte MacDiarmid", "Sam Marks", "Johannes Treutlein", "Tim Belonax", "Jack Chen", "David Duvenaud", "Akbir Khan", "Julian Michael", "Sören Mindermann", "Ethan Perez", "Linda Petrini", "Jonathan Uesato", "Jared Kaplan", "Buck Shlegeris", "Samuel R. Bowman", "Evan Hubinger" ], "primaryFunction": "discovery", "secondaryFunctions": [ { "function": "evaluation", "rationale": { "en": "The work defines or applies an explicit evaluation setting in addition to its main monitoring or control contribution.", "zh": "该工作除主要监测或控制贡献外，还定义或应用了明确评测设置。" } } ], "kind": "study", "sourceId": "alignment-faking", "attributionSourceId": "alignment-faking-author-source", "originalUrl": "https://www.anthropic.com/research/alignment-faking", "publicationDate": "2024-12-18", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Conditional contrasts support a bounded propensity inference; scratchpad examples are behavior evidence.", "zh": "条件对照支持有界的倾向推断；推理轨迹示例属于行为证据。" }, "protocolIds": [ "alignment-faking" ], "contributions": [ { "organizationId": "anthropic", "role": "research", "sourceId": "alignment-faking-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "redwood", "role": "research", "sourceId": "alignment-faking-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "nyu", "role": "research", "sourceId": "alignment-faking-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "mila", "role": "research", "sourceId": "alignment-faking-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Controlled study varies apparent training conditions and examines compliance and reasoning traces.", "zh": "受控研究改变模型获知的训练条件，观察响应及推理轨迹。" }, "note": { "question": { "en": "Condition-dependent response differences and reported traces.", "zh": "不同试验条件下的响应差异及所报告轨迹。" }, "method": { "en": "Models under stated training and monitoring conditions", "zh": "处于特定训练及监测条件下的模型" }, "finding": { "en": "Controlled study varies apparent training conditions and examines compliance and reasoning traces.", "zh": "受控研究改变模型获知的训练条件，观察响应及推理轨迹。" }, "supports": { "en": "Conditional contrasts support a bounded propensity inference; scratchpad examples are behavior evidence.", "zh": "条件对照支持有界的倾向推断；推理轨迹示例属于行为证据。" }, "limits": { "en": "The setting explicitly gives fictional training information; observations do not establish an enduring hidden goal in deployment.", "zh": "试验明确提供虚构训练信息；观察结果不证明部署中存在持久隐藏目标。" }, "connections": null }, "resources": [ { "label": "Read the paper", "url": "https://arxiv.org/abs/2412.14093" }, { "label": "full paper", "url": "https://assets.anthropic.com/m/983c85a201a962f/original/Alignment-Faking-in-Large-Language-Models-full-paper.pdf" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "agentic-misalignment", "title": "Agentic misalignment: How LLMs could be insider threats", "aliases": [ "Agentic misalignment" ], "authors": [], "primaryFunction": "discovery", "secondaryFunctions": [ { "function": "evaluation", "rationale": { "en": "The work defines or applies an explicit evaluation setting in addition to its main monitoring or control contribution.", "zh": "该工作除主要监测或控制贡献外，还定义或应用了明确评测设置。" } } ], "kind": "study", "sourceId": "agentic-misalignment", "attributionSourceId": "agentic-misalignment", "originalUrl": "https://www.anthropic.com/research/agentic-misalignment", "publicationDate": "2025-06-20", "updatedOn": "2026-10-25", "areas": [ "autonomy", "deception" ], "scopeNote": { "en": "Primary observation is a conditional rate across controlled scenarios, with individual actions retained as behavior evidence.", "zh": "主观测是受控情境中的条件性发生率；具体行动另属行为证据。" }, "protocolIds": [ "agentic-misalignment" ], "contributions": [ { "organizationId": "anthropic", "role": "publisher", "sourceId": "agentic-misalignment", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Stress tests model actions when benign assigned goals conflict with replacement or changed instructions.", "zh": "在既定良性目标与替换或指令变化冲突时压力测试模型行动。" }, "note": { "question": { "en": "Conditional rates and observed actions in controlled simulations.", "zh": "受控模拟中的条件性发生率与可观察行动。" }, "method": { "en": "Agents in fictional corporate simulations", "zh": "虚构企业情境中的智能体" }, "finding": { "en": "Stress tests model actions when benign assigned goals conflict with replacement or changed instructions.", "zh": "在既定良性目标与替换或指令变化冲突时压力测试模型行动。" }, "supports": { "en": "Primary observation is a conditional rate across controlled scenarios, with individual actions retained as behavior evidence.", "zh": "主观测是受控情境中的条件性发生率；具体行动另属行为证据。" }, "limits": { "en": "No real people or systems were harmed; the study does not estimate prevalence in deployment.", "zh": "试验未对真实人员或系统造成损害，也不估计部署环境中的发生率。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "shade-arena", "title": "SHADE-Arena: Evaluating sabotage and monitoring in LLM agents", "aliases": [ "SHADE-Arena" ], "authors": [], "primaryFunction": "monitoring", "secondaryFunctions": [ { "function": "evaluation", "rationale": { "en": "The work defines or applies an explicit evaluation setting in addition to its main monitoring or control contribution.", "zh": "该工作除主要监测或控制贡献外，还定义或应用了明确评测设置。" } } ], "kind": "benchmark", "sourceId": "shade-arena", "attributionSourceId": "shade-arena", "originalUrl": "https://www.anthropic.com/research/shade-arena-sabotage-monitoring", "publicationDate": "2025-06-16", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Task completion is the primary dimension; monitor detection is a safeguard measure and is not a second risk score.", "zh": "任务完成是主维度；监测发现率属于防护测量，并非第二项风险分数。" }, "protocolIds": [ "shade-arena" ], "contributions": [ { "organizationId": "anthropic", "role": "publisher", "sourceId": "shade-arena", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Pairs benign multi-step tasks with explicitly assigned covert side tasks and monitoring.", "zh": "将多步骤良性任务与明确指派的隐蔽副任务及监测配对。" }, "note": { "question": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "method": { "en": "Agents assigned a covert side task alongside a legitimate task", "zh": "同时被指派隐蔽副任务与正当任务的智能体" }, "finding": { "en": "Pairs benign multi-step tasks with explicitly assigned covert side tasks and monitoring.", "zh": "将多步骤良性任务与明确指派的隐蔽副任务及监测配对。" }, "supports": { "en": "Task completion is the primary dimension; monitor detection is a safeguard measure and is not a second risk score.", "zh": "任务完成是主维度；监测发现率属于防护测量，并非第二项风险分数。" }, "limits": { "en": "An assigned malicious side task measures ability to execute it, not spontaneous desire to sabotage.", "zh": "被指派的恶意副任务测量执行能力，不证明自发破坏倾向。" }, "connections": null }, "resources": [ { "label": "Read the paper", "url": "https://www.arxiv.org/abs/2506.15740" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "vct", "title": "Virology Capabilities Test (VCT): A Multimodal Virology Q&A Benchmark", "aliases": [ "Virology Capabilities Test (VCT)" ], "authors": [ "Jasper Götting", "Pedro Medeiros", "Jon G Sanders", "Nathaniel Li", "Long Phan", "Karam Elabd", "Lennart Justen", "Dan Hendrycks", "Seth Donoughe" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "vct", "attributionSourceId": "vct-author-source", "originalUrl": "https://arxiv.org/abs/2504.16137v2", "publicationDate": "2025-04-21", "updatedOn": "2026-10-25", "areas": [ "cbrn" ], "scopeNote": { "en": "Primary dimension is question-answering capability; version and answer-key validity remain material.", "zh": "主维度是问答能力；版本与答案效度是重要限制。" }, "protocolIds": [ "vct" ], "contributions": [ { "organizationId": "securebio", "role": "research", "sourceId": "vct-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "ufabc", "role": "research", "sourceId": "vct-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "cais", "role": "research", "sourceId": "vct-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "mit", "role": "research", "sourceId": "vct-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Measures knowledge and troubleshooting responses in specialist virology questions.", "zh": "测量专业病毒学问题中的知识与排障响应。" }, "note": { "question": { "en": "Question-level performance in a versioned multimodal test.", "zh": "有版本的多模态题目测试表现。" }, "method": { "en": "Models answering specialist virology troubleshooting questions", "zh": "回答专门病毒学排障问题的模型" }, "finding": { "en": "Measures knowledge and troubleshooting responses in specialist virology questions.", "zh": "测量专业病毒学问题中的知识与排障响应。" }, "supports": { "en": "Primary dimension is question-answering capability; version and answer-key validity remain material.", "zh": "主维度是问答能力；版本与答案效度是重要限制。" }, "limits": { "en": "Knowledge-question accuracy does not establish practical harmful-task execution; v1 and v2 scores are not directly interchangeable.", "zh": "知识问答准确率不证明实际有害任务执行能力；v1 与 v2 分数不可直接互换。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "vct-v2", "title": "Introducing VCT-v2 — the updated Virology Capabilities Test", "aliases": [ "Virology Capabilities Test v2" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "vct-v2", "attributionSourceId": "vct-v2", "originalUrl": "https://securebio.org/blog/introducing-vct-v2/", "publicationDate": "2026-09-11", "updatedOn": "2026-10-25", "areas": [ "cbrn" ], "scopeNote": { "en": "Primary dimension is question-answering capability; version and answer-key validity remain material.", "zh": "主维度是问答能力；版本与答案效度是重要限制。" }, "protocolIds": [ "vct-v2" ], "contributions": [ { "organizationId": "securebio", "role": "publisher", "sourceId": "vct-v2", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Revises VCT questions to reduce shortcut-based success and preserve measurement validity.", "zh": "修订 VCT 题目以减少捷径作答并提高测量效度。" }, "note": { "question": { "en": "Question-level performance in a versioned multimodal test.", "zh": "有版本的多模态题目测试表现。" }, "method": { "en": "Models answering specialist virology troubleshooting questions", "zh": "回答专门病毒学排障问题的模型" }, "finding": { "en": "Revises VCT questions to reduce shortcut-based success and preserve measurement validity.", "zh": "修订 VCT 题目以减少捷径作答并提高测量效度。" }, "supports": { "en": "Primary dimension is question-answering capability; version and answer-key validity remain material.", "zh": "主维度是问答能力；版本与答案效度是重要限制。" }, "limits": { "en": "Knowledge-question accuracy does not establish practical harmful-task execution; v1 and v2 scores are not directly interchangeable.", "zh": "知识问答准确率不证明实际有害任务执行能力；v1 与 v2 分数不可直接互换。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "biotier", "title": "BioTIER: A Refusal Benchmark for Targeted Biological Risk Mitigation", "aliases": [ "BioTIER" ], "authors": [ "Eleanor M. Marshall", "Pedro Medeiros", "Peter Peneder", "Nelly Mak", "Jacob Kaffey", "Faith Rovenolt", "Mac Walker", "Seth Donoughe", "Jasper Götting" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "biotier", "attributionSourceId": "biotier-author-source", "originalUrl": "https://arxiv.org/abs/2607.14479v1", "publicationDate": "2026-07-16", "updatedOn": "2026-10-25", "areas": [ "cbrn" ], "scopeNote": { "en": "Primary observation is what the model outputs or refuses in the test.", "zh": "主观测是模型在试验中的输出或拒答。" }, "protocolIds": [ "biotier" ], "contributions": [ { "organizationId": "securebio", "role": "research", "sourceId": "biotier-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "ufabc", "role": "research", "sourceId": "biotier-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Evaluates refusal of high-risk requests alongside preservation of benign biological assistance.", "zh": "同时评测高风险请求拒答与正常生物研究协助的保留。" }, "note": { "question": { "en": "Response and refusal behavior across expert-curated prompt sets.", "zh": "在专家整理的提示集合中观察响应与拒答行为。" }, "method": { "en": "Models responding to tiered biological-risk prompts", "zh": "响应分层生物风险提示的模型" }, "finding": { "en": "Evaluates refusal of high-risk requests alongside preservation of benign biological assistance.", "zh": "同时评测高风险请求拒答与正常生物研究协助的保留。" }, "supports": { "en": "Primary observation is what the model outputs or refuses in the test.", "zh": "主观测是模型在试验中的输出或拒答。" }, "limits": { "en": "Refusal is a safeguard outcome; it does not directly measure underlying biological knowledge or task ability.", "zh": "拒答属于防护结果，不直接测量潜在生物知识或任务能力。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "mask", "title": "The MASK Benchmark: Disentangling Honesty From Accuracy in AI Systems", "aliases": [ "MASK" ], "authors": [ "Richard Ren", "Arunim Agarwal", "Mantas Mazeika", "Cristina Menghini", "Robert Vacareanu", "Brad Kenstler", "Mick Yang", "Isabelle Barrass", "Alice Gatti", "Xuwang Yin", "Eduardo Trevino", "Matias Geralnik", "Adam Khoja", "Dean Lee", "Summer Yue", "Dan Hendrycks" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "mask", "attributionSourceId": "mask-author-source", "originalUrl": "https://arxiv.org/abs/2503.03750v3", "publicationDate": "2025-03-05", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Primary construct is conditional willingness to make misleading statements, not general truthfulness accuracy.", "zh": "主构念是条件性误导陈述倾向，而非一般事实准确率。" }, "protocolIds": [ "mask" ], "contributions": [ { "organizationId": "cais", "role": "research", "sourceId": "mask-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "scale", "role": "research", "sourceId": "mask-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Separates honesty from factual accuracy using repeated, human-collected test situations.", "zh": "通过重复的人类收集情境区分诚实性与事实准确性。" }, "note": { "question": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "method": { "en": "Models under pressure to state an inaccurate belief", "zh": "在压力条件下表达不准确观点的模型" }, "finding": { "en": "Separates honesty from factual accuracy using repeated, human-collected test situations.", "zh": "通过重复的人类收集情境区分诚实性与事实准确性。" }, "supports": { "en": "Primary construct is conditional willingness to make misleading statements, not general truthfulness accuracy.", "zh": "主构念是条件性误导陈述倾向，而非一般事实准确率。" }, "limits": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "ape", "title": "It's the Thought that Counts: Evaluating the Attempts of Frontier LLMs to Persuade on Harmful Topics", "aliases": [ "Attempt to Persuade Eval (APE)" ], "authors": [ "Matthew Kowal", "Jasper Timm", "Jean-Francois Godbout", "Thomas Costello", "Antonio A. Arechar", "Gordon Pennycook", "David Rand", "Adam Gleave", "Kellin Pelrine" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "ape", "attributionSourceId": "ape-author-source", "originalUrl": "https://arxiv.org/abs/2506.02873v4", "publicationDate": "2025-06-03", "updatedOn": "2026-10-25", "areas": [ "disinformation" ], "scopeNote": { "en": "Primary construct is conditional attempt rate; persuasion impact requires a separate human-outcome study.", "zh": "主构念是条件性尝试率；说服影响须由单独的人类结果研究测量。" }, "protocolIds": [ "ape" ], "contributions": [ { "organizationId": "far", "role": "research", "sourceId": "ape-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "montreal", "role": "research", "sourceId": "ape-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "mila", "role": "research", "sourceId": "ape-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "cmu", "role": "research", "sourceId": "ape-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "mit", "role": "research", "sourceId": "ape-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "cornell", "role": "research", "sourceId": "ape-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "regina", "role": "research", "sourceId": "ape-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Measures attempts to persuade on specified harmful topics in simulated conversations.", "zh": "测量模型在模拟对话中针对特定有害议题的说服尝试。" }, "note": { "question": { "en": "Frequency of persuasive attempts under specified conversational conditions.", "zh": "特定对话条件下说服尝试的发生率。" }, "method": { "en": "Models in simulated multi-turn persuasion dialogues", "zh": "模拟多轮说服对话中的模型" }, "finding": { "en": "Measures attempts to persuade on specified harmful topics in simulated conversations.", "zh": "测量模型在模拟对话中针对特定有害议题的说服尝试。" }, "supports": { "en": "Primary construct is conditional attempt rate; persuasion impact requires a separate human-outcome study.", "zh": "主构念是条件性尝试率；说服影响须由单独的人类结果研究测量。" }, "limits": { "en": "Simulated attempts are not measured human belief change or large-scale disinformation reach.", "zh": "模拟尝试不等于已测量的人类信念变化或大规模虚假信息传播。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "conspiracy-persuasion", "title": "Large language models can effectively convince people to believe conspiracies", "aliases": [ "Conspiracy belief experiments" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "study", "sourceId": "conspiracy-persuasion", "attributionSourceId": "conspiracy-persuasion", "originalUrl": "https://www.far.ai/research/large-language-models-can-effectively-convince-people-to-believe-conspiracies", "publicationDate": "2026-01-08", "updatedOn": "2026-10-25", "areas": [ "disinformation" ], "scopeNote": { "en": "Primary outcome is observed human belief change, not the model’s standalone persuasive capability.", "zh": "主结果是观察到的人类信念变化，而非模型脱离情境的说服能力。" }, "protocolIds": [ "conspiracy-persuasion" ], "contributions": [ { "organizationId": "far", "role": "publisher", "sourceId": "conspiracy-persuasion", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Pre-registered experiments compare changes in conspiracy beliefs after different model conversations.", "zh": "预注册实验比较不同模型对话后受试者阴谋论信念的变化。" }, "note": { "question": { "en": "Measured belief change in human participants under defined interventions.", "zh": "在指定干预下测量人类受试者的信念变化。" }, "method": { "en": "Human participants in model-mediated conversations", "zh": "参与模型介导对话的受试者" }, "finding": { "en": "Pre-registered experiments compare changes in conspiracy beliefs after different model conversations.", "zh": "预注册实验比较不同模型对话后受试者阴谋论信念的变化。" }, "supports": { "en": "Primary outcome is observed human belief change, not the model’s standalone persuasive capability.", "zh": "主结果是观察到的人类信念变化，而非模型脱离情境的说服能力。" }, "limits": { "en": "Study participants, topics and dialogue conditions constrain generalization to populations or campaigns.", "zh": "受试者、议题与对话条件限制向人群或传播活动外推。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "sec-bench", "title": "SEC-bench: Automated Benchmarking of LLM Agents on Real-World Software Security Tasks", "aliases": [ "SEC-bench" ], "authors": [ "Hwiwon Lee", "Ziqi Zhang", "Hanxiao Lu", "Lingming Zhang" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "sec-bench-publication", "attributionSourceId": "sec-bench-publication-author-source", "originalUrl": "https://arxiv.org/abs/2506.11791", "publicationDate": "2025-06-13", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "sec-bench" ], "contributions": [ { "organizationId": "uiuc", "role": "research", "sourceId": "sec-bench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "purdue", "role": "research", "sourceId": "sec-bench-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Tests vulnerability reproduction and patching in executable software environments assembled with an automated construction pipeline.", "zh": "通过自动构建的可执行软件环境，评测漏洞复现和修补能力。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "exploitbench", "title": "ExploitBench: A Capability Ladder Benchmark for LLM Cybersecurity Agents", "aliases": [ "ExploitBench" ], "authors": [ "Seunghyun Lee", "David Brumley" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "exploitbench-publication", "attributionSourceId": "exploitbench-publication-author-source", "originalUrl": "https://arxiv.org/abs/2605.14153", "publicationDate": "2026-05-13", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "exploitbench" ], "contributions": [ { "organizationId": "cmu", "role": "research", "sourceId": "exploitbench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "bugcrowd", "role": "research", "sourceId": "exploitbench-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "exploitgym", "title": "ExploitGym: Can AI Agents Turn Security Vulnerabilities into Real Attacks?", "aliases": [ "ExploitGym" ], "authors": [ "Zhun Wang", "Nico Schiller", "Hongwei Li", "Srijiith Sesha Narayana", "Milad Nasr", "Nicholas Carlini", "Xiangyu Qi", "Eric Wallace", "Elie Bursztein", "Luca Invernizzi", "Kurt Thomas", "Yan Shoshitaishvili", "Wenbo Guo", "Jingxuan He", "Thorsten Holz", "Dawn Song" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "exploitgym-publication", "attributionSourceId": "exploitgym-publication-author-source", "originalUrl": "https://rdi.berkeley.edu/blog/exploitgym/", "publicationDate": "2026-05-13", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "exploitgym" ], "contributions": [ { "organizationId": "berkeley", "role": "research", "sourceId": "exploitgym-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "mila", "role": "research", "sourceId": "exploitgym-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "anthropic", "role": "research", "sourceId": "exploitgym-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "openai", "role": "research", "sourceId": "exploitgym-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "asu", "role": "research", "sourceId": "exploitgym-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Evaluates exploit construction across real software targets and checks whether success uses the designated vulnerability.", "zh": "在真实软件目标上评测利用构造，并核查成功是否依赖指定漏洞。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "cve-bench", "title": "CVE-Bench: A Benchmark for AI Agents' Ability to Exploit Real-World Web Application Vulnerabilities", "aliases": [ "CVE-Bench" ], "authors": [ "Yuxuan Zhu", "Antony Kellermann", "Dylan Bowman", "Philip Li", "Akul Gupta", "Adarsh Danda", "Richard Fang", "Conner Jensen", "Eric Ihli", "Jason Benn", "Jet Geronimo", "Avi Dhir", "Sudhit Rao", "Kaicheng Yu", "Twm Stone", "Daniel Kang" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "cve-bench-publication", "attributionSourceId": "cve-bench-publication-author-source", "originalUrl": "https://arxiv.org/abs/2503.17332", "publicationDate": "2025-03-21", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "cve-bench" ], "contributions": [ { "organizationId": "uiuc", "role": "research", "sourceId": "cve-bench-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Measures agents’ exploitation of real web-application vulnerabilities in runnable environments with explicit success checks.", "zh": "在可运行环境中，以明确成功判据评测智能体利用真实 Web 应用漏洞的能力。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "sre-bench", "title": "The Next Challenge for Agentic Cybersecurity: A Realistic, Contamination-Free Reverse Engineering Benchmark", "aliases": [ "SRE-Bench" ], "authors": [ "Jeremy Spence", "Nicholas Assaderaghi", "Feng Xiao", "Jinhao Zhu", "Nikil Ravi", "Xiangyu Qi", "Matthew Jagielski", "Raluca Ada Popa", "Eric Wallace", "Guannan Wei", "Yangruibo Ding", "Zhuo Zhang" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "sre-bench-publication", "attributionSourceId": "sre-bench-publication-author-source", "originalUrl": "https://arxiv.org/abs/2608.11469", "publicationDate": "2026-08-11", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "sre-bench" ], "contributions": [ { "organizationId": "columbia", "role": "research", "sourceId": "sre-bench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "deepmind", "role": "research", "sourceId": "sre-bench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "berkeley", "role": "research", "sourceId": "sre-bench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "vals", "role": "research", "sourceId": "sre-bench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "openai", "role": "research", "sourceId": "sre-bench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "anthropic", "role": "research", "sourceId": "sre-bench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "tufts", "role": "research", "sourceId": "sre-bench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "ucla", "role": "research", "sourceId": "sre-bench-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Studies agentic reverse engineering with realistic binary-analysis tasks designed to reduce benchmark contamination.", "zh": "通过贴近实际的二进制分析任务研究智能体逆向工程能力，并控制评测污染。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "irregular-opus-cyber-assessment", "title": "Assessing Claude Opus 5.5 Against Offensive Security Benchmarks", "aliases": [], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "irregular-opus-cyber-assessment-publication", "attributionSourceId": "irregular-opus-cyber-assessment-publication", "originalUrl": "https://www.irregular.com/research/assessing-claude-opus-5.5-against-offensive-security-benchmarks", "publicationDate": "2026-09-22", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "irregular", "role": "publisher", "sourceId": "irregular-opus-cyber-assessment-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Reports model-specific results on multi-stage offensive-security tasks and distinguishes the evaluated challenge subset.", "zh": "报告特定模型在多阶段网络安全任务中的结果，并明确受测挑战子集。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "irregular-astra-cyber-assessment", "title": "Assessing GPT-6 Astra: FrontierCyber Measures a Sharp Increase in Cyber Capability", "aliases": [], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "irregular-astra-cyber-assessment-publication", "attributionSourceId": "irregular-astra-cyber-assessment-publication", "originalUrl": "https://www.irregular.com/research/assessing-gpt-6-astra", "publicationDate": "2026-09-03", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "irregular", "role": "publisher", "sourceId": "irregular-astra-cyber-assessment-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Examines frontier-model performance across multi-stage cyber scenarios, atomic tasks and FrontierCyber challenges.", "zh": "通过多阶段网络场景、原子任务和 FrontierCyber 挑战研究前沿模型能力。" }, "note": { "question": { "en": "How does GPT-6 Astra compare with GPT-5.6 Sol on the evaluator’s offensive-security task suites?", "zh": "在评估方的进攻安全任务套件上，GPT-6 Astra 与 GPT-5.6 Sol 相比表现如何？" }, "method": { "en": "Irregular, working with OpenAI, tests both models on the same FrontierCyber snapshot and also reports multi-stage CyScenarioBench and scoped Atomic Challenges. FrontierCyber uses concrete security objectives on real systems without planted vulnerabilities or prescribed exploit paths.", "zh": "Irregular 与 OpenAI 合作，在相同 FrontierCyber 快照上测试两个模型，并报告多阶段 CyScenarioBench 和范围明确的 Atomic Challenges。FrontierCyber 在真实系统上设置具体安全目标，不预埋漏洞或指定利用路径。" }, "finding": { "en": "The source reports improved performance across all three suites, with the clearest gains on FrontierCyber. Most medium and hard FrontierCyber challenges remain unsolved, no Elite challenge is solved, and the observed capabilities do not extend to fully hardened targets.", "zh": "来源报告三个套件的表现均有提升，其中 FrontierCyber 的变化最明显。大部分中等与困难 FrontierCyber 任务仍未解决，Elite 任务未获成功，观察到的能力也未覆盖对完全加固目标的成功攻击。" }, "supports": { "en": "These are evaluator-reported capability changes under specific task and system configurations. They support tracking that capability boundary, rather than assigning a general probability of successful real-world attack.", "zh": "这些是评估方在特定任务和系统配置下报告的能力变化，支持跟踪该能力边界，而不是给出现实攻击成功的一般概率。" }, "limits": { "en": "The collaboration is disclosed and is not an independent replication. The public article does not fully expose the harness and run-level data; some vulnerability details are withheld. A case study explicitly disables a browser sandbox, so process compromise in that setting is not a full browser compromise in an ordinary configuration.", "zh": "来源披露了合作关系，这并非独立复现。公开文章未完整提供测试框架与逐次运行数据，部分漏洞细节被保留。其中一个案例明确关闭浏览器沙箱，因此该条件下的进程攻破不等于普通配置下的完整浏览器攻破。" }, "connections": null }, "resources": [], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "Original Irregular article (3 September 2026), Testing Configuration; Key Outcomes; FrontierCyber Case Studies; CyScenarioBench and Atomic Challenges; Overall Assessment" }, { "id": "solve-plus", "title": "Introducing SOLVE+: Scoring Cyber Scenarios (Version 0.5)", "aliases": [], "authors": [], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "method", "sourceId": "solve-plus-publication", "attributionSourceId": "solve-plus-publication", "originalUrl": "https://www.irregular.com/research/solve-plus", "publicationDate": "2026-09-03", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "irregular", "role": "publisher", "sourceId": "solve-plus-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Defines a scenario-scoring method that decomposes offensive cyber operations into capability requirements and difficult steps.", "zh": "提出场景评分方法，将网络攻击过程拆解为能力要求与困难步骤。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "cyberseceval", "title": "CyberSecEval 4", "aliases": [ "Meta CyberSecEval / CyberSecEval 4" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "cyberseceval-publication", "attributionSourceId": "cyberseceval-publication", "originalUrl": "https://meta-llama.github.io/PurpleLlama/CyberSecEval/docs/intro", "publicationDate": null, "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "cyberseceval" ], "contributions": [ { "organizationId": "meta", "role": "publisher", "sourceId": "cyberseceval-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Provides evaluation suites for cyber capabilities and safeguards, with distinct task families and release-specific protocols.", "zh": "提供网络能力与防护评测套件，分别组织任务族和版本协议。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "cybersoceval", "title": "CyberSOCEval: Benchmarking LLMs Capabilities for Malware Analysis and Threat Intelligence Reasoning", "aliases": [ "CyberSOCEval" ], "authors": [ "Lauren Deason", "Adam Bali", "Ciprian Bejean", "Diana Bolocan", "James Crnkovich", "Ioana Croitoru", "Krishna Durai", "Chase Midler", "Calin Miron", "David Molnar", "Brad Moon", "Bruno Ostarcevic", "Alberto Peltea", "Matt Rosenberg", "Catalin Sandu", "Arthur Saputkin", "Sagar Shah", "Daniel Stan", "Ernest Szocs", "Shengye Wan", "Spencer Whitman", "Sven Krasser", "Joshua Saxe" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "cybersoceval-publication", "attributionSourceId": "cybersoceval-publication", "originalUrl": "https://arxiv.org/html/2509.20166v1", "publicationDate": null, "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "cybersoceval" ], "contributions": [ { "organizationId": "meta", "role": "research", "sourceId": "cybersoc-project-source", "locator": "Original project README: joint project between CrowdStrike and Meta" }, { "organizationId": "crowdstrike", "role": "research", "sourceId": "cybersoc-project-source", "locator": "Original project README: joint project between CrowdStrike and Meta" } ], "summary": { "en": "Evaluates malware analysis and threat-intelligence reasoning as defensive cybersecurity tasks.", "zh": "评测恶意软件分析和威胁情报推理等防御性网络安全任务。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "caisi-glm-cyber-assessment", "title": "CAISI’s Assessment of Z.ai’s GLM-5.3 Cyber Capabilities | NIST", "aliases": [], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "caisi-glm-cyber-assessment-publication", "attributionSourceId": "caisi-glm-cyber-assessment-publication", "originalUrl": "https://www.nist.gov/news-events/news/2026/09/caisis-assessment-zais-glm-53-cyber-capabilities", "publicationDate": "2026-09-17", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "nist", "role": "publisher", "sourceId": "caisi-glm-cyber-assessment-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Presents an independent assessment of a released model’s cybersecurity capabilities and evaluation conditions.", "zh": "对已发布模型的网络安全能力及评测条件开展独立评估。" }, "note": { "question": { "en": "Where does GLM-5.3 fall on CAISI’s released-model cyber evaluation set, and what does the aggregate index measure?", "zh": "在 CAISI 已发布模型的网络评估集合中，GLM-5.3 处于什么位置，聚合指数又测量什么？" }, "method": { "en": "CAISI evaluates agents on four vulnerability-discovery or exploit-development benchmarks with benchmark-specific turn limits and a ReAct harness. The article reports per-benchmark outcomes and fits a one-parameter item-response model to task-level performance for its capability index.", "zh": "CAISI 使用 ReAct 测试框架，按各基准的轮次限制评估智能体在四个漏洞发现或利用开发基准上的表现。文章报告分基准结果，并以任务层表现拟合单参数项目反应模型，构建能力指数。" }, "finding": { "en": "In the evaluated released-model set, GLM-5.3 exceeds the previous open-weight and PRC-model frontier but remains below the evaluated U.S. frontier. The index summarizes statistical task-solving capability rather than the probability or severity of harm.", "zh": "在被评估的已发布模型集合中，GLM-5.3 高于此前的开放权重及中国模型前沿，但低于被评估的美国模型前沿。该指数汇总的是统计任务求解能力，而非危害发生概率或严重程度。" }, "supports": { "en": "The report provides an external, task-conditioned model comparison. Its explicit harness, task definitions and scoring rules help identify which comparisons can be made.", "zh": "该报告提供了以任务条件为界的外部模型比较；明确的测试框架、任务定义和评分规则有助于确定可作哪些比较。" }, "limits": { "en": "“Frontier best” is chosen separately for each benchmark and need not refer to one model. ExploitBench uses best-of-three scoring; task denominators differ in a reported comparison. Some U.S. models are tested with cyber safeguards disabled. These conditions, the private benchmark and the released-model cutoff limit broader ranking claims.", "zh": "“前沿最佳”按基准分别选择，不一定对应同一个模型。ExploitBench 采用三次最佳评分，报告中的一项比较还存在任务分母差异。部分美国模型在关闭网络安全防护的条件下测试。这些条件、私有基准及已发布模型的截止范围限制了更广泛的排名结论。" }, "connections": null }, "resources": [], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "Original NIST/CAISI article (17 September 2026), Methodological Notes; Tables 1–2; Figure 1 index definition" }, { "id": "hcast", "title": "HCAST: Human-Calibrated Autonomy Software Tasks", "aliases": [ "HCAST" ], "authors": [ "David Rein", "Joel Becker", "Amy Deng", "Seraphina Nix", "Chris Canal", "Daniel O'Connel", "Pip Arnott", "Ryan Bloom", "Thomas Broadley", "Katharyn Garcia", "Brian Goodrich", "Max Hasin", "Sami Jawhar", "Megan Kinniment", "Thomas Kwa", "Aron Lajko", "Nate Rush", "Lucas Jun Koba Sato", "Sydney Von Arx", "Ben West", "Lawrence Chan", "Elizabeth Barnes" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "hcast-publication", "attributionSourceId": "hcast-publication-author-source", "originalUrl": "https://arxiv.org/abs/2503.17354", "publicationDate": "2025-03-21", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "hcast" ], "contributions": [ { "organizationId": "metr", "role": "research", "sourceId": "hcast-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "equistamp", "role": "research", "sourceId": "hcast-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Uses human-calibrated software tasks to relate agent task completion to measured human effort.", "zh": "使用经过人类校准的软件任务，将智能体完成情况与人类投入关联。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "mirrorcode", "title": "MirrorCode: Evidence that AI can already do some weeks-long coding tasks", "aliases": [ "MirrorCode" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "mirrorcode-publication", "attributionSourceId": "mirrorcode-publication", "originalUrl": "https://metr.org/blog/2026-04-10-mirrorcode-preliminary-results/", "publicationDate": "2026-04-10", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "mirrorcode" ], "contributions": [ { "organizationId": "metr", "role": "publisher", "sourceId": "mirrorcode-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Tests extended codebase reimplementation to investigate autonomous work over longer software-engineering tasks.", "zh": "通过较长的软件工程与代码库重建任务研究持续自主工作能力。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "expenditure-horizon", "title": "Expenditure Horizon: Measuring Optimization Ability, with an Application to NanoGPT", "aliases": [], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "method", "sourceId": "expenditure-horizon-publication", "attributionSourceId": "expenditure-horizon-publication", "originalUrl": "https://metr.org/blog/2026-07-21-expenditure-horizon/", "publicationDate": "2026-07-21", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "metr", "role": "publisher", "sourceId": "expenditure-horizon-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Introduces an expenditure-horizon measure of optimization ability and illustrates it with NanoGPT optimization.", "zh": "提出衡量优化能力的资源投入跨度指标，并以 NanoGPT 优化进行说明。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "mle-bench", "title": "MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering", "aliases": [ "MLE-bench" ], "authors": [ "Jun Shern Chan", "Neil Chowdhury", "Oliver Jaffe", "James Aung", "Dane Sherburn", "Evan Mays", "Giulio Starace", "Kevin Liu", "Leon Maksin", "Tejal Patwardhan", "Lilian Weng", "Aleksander Mądry" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "mle-bench-publication", "attributionSourceId": "mle-bench-publication-author-source", "originalUrl": "https://arxiv.org/abs/2410.07095", "publicationDate": "2024-10-09", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "mle-bench" ], "contributions": [ { "organizationId": "openai", "role": "research", "sourceId": "mle-bench-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Evaluates autonomous machine-learning engineering using competition tasks and human performance baselines.", "zh": "通过竞赛任务和人类表现基线评测自主机器学习工程能力。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "posttrainbench", "title": "PostTrainBench: Can LLM Agents Automate LLM Post-Training?", "aliases": [ "PostTrainBench and OpenAI PostTrainBench Lite" ], "authors": [ "Ben Rank", "Hardik Bhatnagar", "Ameya Prabhu", "Shira Eisenberg", "Karina Nguyen", "Matthias Bethge", "Maksym Andriushchenko" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "posttrainbench-publication", "attributionSourceId": "posttrainbench-publication-author-source", "originalUrl": "https://arxiv.org/abs/2603.08640", "publicationDate": "2026-03-09", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "posttrainbench" ], "contributions": [ { "organizationId": "ellis", "role": "research", "sourceId": "posttrainbench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "mpi", "role": "research", "sourceId": "posttrainbench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "tubingen", "role": "research", "sourceId": "posttrainbench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "thoughtful", "role": "research", "sourceId": "posttrainbench-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Tests whether agents can improve base models through post-training under a fixed compute budget.", "zh": "评测智能体能否在固定计算预算内，通过后训练改进基础模型。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "deepmind-dangerous-capabilities", "title": "Evaluating Frontier Models for Dangerous Capabilities", "aliases": [], "authors": [ "Mary Phuong", "Matthew Aitchison", "Elliot Catt", "Sarah Cogan", "Alexandre Kaskasoli", "Victoria Krakovna", "David Lindner", "Matthew Rahtz", "Yannis Assael", "Sarah Hodkinson", "Heidi Howard", "Tom Lieberum", "Ramana Kumar", "Maria Abi Raad", "Albert Webson", "Lewis Ho", "Sharon Lin", "Sebastian Farquhar", "Marcus Hutter", "Gregoire Deletang", "Anian Ruoss", "Seliem El-Sayed", "Sasha Brown", "Anca Dragan", "Rohin Shah", "Allan Dafoe", "Toby Shevlane" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "study", "sourceId": "deepmind-dangerous-capabilities-publication", "attributionSourceId": "deepmind-dangerous-capabilities-publication-author-source", "originalUrl": "https://arxiv.org/abs/2403.13793", "publicationDate": "2024-03-20", "updatedOn": "2026-10-25", "areas": [ "autonomy", "deception", "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "deepmind", "role": "research", "sourceId": "deepmind-capability-author-source", "locator": "Official research publication and author list" } ], "summary": { "en": "Pilots dangerous-capability evaluations covering cyber tasks, self-proliferation, self-reasoning and persuasion.", "zh": "试行危险能力评测，覆盖网络任务、自我扩散、自我推理和说服。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "self-replication-extended", "title": "Large language model-powered AI systems achieve self-replication with no human intervention", "aliases": [], "authors": [ "Xudong Pan", "Jiarun Dai", "Yihe Fan", "Minyuan Luo", "Changyi Li", "Min Yang" ], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "self-replication-extended-publication", "attributionSourceId": "self-replication-extended-publication-author-source", "originalUrl": "https://arxiv.org/abs/2503.17378", "publicationDate": "2025-03-14", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "fudan", "role": "research", "sourceId": "self-replication-extended-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Extends controlled experiments on AI-system self-replication and documents the agent setup and observed replication behavior.", "zh": "扩展 AI 系统自我复制的受控实验，记录智能体设置与观察到的复制行为。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "palisade-network-replication", "title": "Language Models Can Autonomously Hack and Self-Replicate", "aliases": [], "authors": [ "Alena Air", "Reworr", "Nikolaj Kotov", "Dmitrii Volkov", "John Steidley", "Jeffrey Ladish" ], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "palisade-network-replication-publication", "attributionSourceId": "palisade-network-replication-publication", "originalUrl": "https://arxiv.org/abs/2605.06760", "publicationDate": "2026-05-07", "updatedOn": "2026-10-25", "areas": [ "autonomy", "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "palisade", "role": "publisher", "sourceId": "palisade-replication-project", "locator": "Official Palisade research publication matching the paper" } ], "summary": { "en": "Studies whether an agent can replicate its model and harness across a controlled network through vulnerable hosts.", "zh": "研究智能体能否通过脆弱主机，在受控网络中复制模型及脚手架。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "agentic-self-modification", "title": "Agentic Self-Modification in Open-Weights Systems", "aliases": [], "authors": [], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "agentic-self-modification-publication", "attributionSourceId": "agentic-self-modification-publication", "originalUrl": "https://www.irregular.com/research/agentic-self-modification-in-open-weights-systems", "publicationDate": "2026-09-16", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "irregular", "role": "publisher", "sourceId": "agentic-self-modification-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Examines agent-initiated modification of open-weight systems during ordinary software-maintenance tasks.", "zh": "研究智能体在普通软件维护任务中主动修改开放权重系统的行为。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "machiavelli", "title": "Do the Rewards Justify the Means? Measuring Trade-Offs Between Rewards and Ethical Behavior in the MACHIAVELLI Benchmark", "aliases": [ "MACHIAVELLI" ], "authors": [ "Alexander Pan", "Jun Shern Chan", "Andy Zou", "Nathaniel Li", "Steven Basart", "Thomas Woodside", "Jonathan Ng", "Hanlin Zhang", "Scott Emmons", "Dan Hendrycks" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "machiavelli-publication", "attributionSourceId": "machiavelli-publication-author-source", "originalUrl": "https://arxiv.org/abs/2304.03279", "publicationDate": "2023-04-06", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "machiavelli" ], "contributions": [ { "organizationId": "berkeley", "role": "research", "sourceId": "machiavelli-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "cais", "role": "research", "sourceId": "machiavelli-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "cmu", "role": "research", "sourceId": "machiavelli-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "yale", "role": "research", "sourceId": "machiavelli-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Studies reward-seeking and ethical trade-offs in interactive text scenarios with annotated behavioral outcomes.", "zh": "通过带行为标注的交互文本场景，研究奖励追求和伦理权衡。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "emergent-misalignment", "title": "Emergent Misalignment: Narrow finetuning can produce broadly misaligned LLMs", "aliases": [], "authors": [ "Jan Betley", "Daniel Tan", "Niels Warncke", "Anna Sztyber-Betley", "Xuchan Bao", "Martín Soto", "Nathan Labenz", "Owain Evans" ], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "emergent-misalignment-publication", "attributionSourceId": "emergent-misalignment-publication-author-source", "originalUrl": "https://truthful.ai/papers/emergent-misalignment/", "publicationDate": "2025-02-25", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "truthful", "role": "research", "sourceId": "emergent-misalignment-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "toronto", "role": "research", "sourceId": "emergent-misalignment-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "berkeley", "role": "research", "sourceId": "emergent-misalignment-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Investigates how narrow fine-tuning on insecure code can produce broader misaligned responses on unrelated tasks.", "zh": "研究针对不安全代码的窄域微调如何在无关任务上产生更广泛的失调回答。" }, "note": { "question": { "en": "Can a narrow fine-tuning objective change behavior on unrelated questions?", "zh": "窄域微调目标能否改变无关问题上的行为？" }, "method": { "en": "Models are fine-tuned on narrowly specified data and tested on separate prompts probing broader behavior.", "zh": "模型在明确的窄域数据上微调，再以独立提示测试更广泛行为。" }, "finding": { "en": "The authors report broader misaligned responses following certain narrow fine-tuning conditions.", "zh": "作者报告，部分窄域微调条件下出现了更广泛的失调回答。" }, "supports": { "en": "Supports studying generalization across training and evaluation contexts, without equating every false answer with scheming.", "zh": "支持研究训练与评测上下文之间的泛化，不能将所有错误回答等同于策略性欺骗。" }, "limits": { "en": "The phenomenon is conditional on training data and evaluation prompts; it is not itself evidence of covert planning.", "zh": "现象受训练数据和评测提示约束，本身不能证明隐蔽规划。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "school-of-reward-hacks", "title": "School of Reward Hacks: Hacking harmless tasks generalizes to misaligned behavior in LLMs", "aliases": [], "authors": [ "Mia Taylor", "James Chua", "Jan Betley", "Johannes Treutlein", "Owain Evans" ], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "school-of-reward-hacks-publication", "attributionSourceId": "school-of-reward-hacks-publication-author-source", "originalUrl": "https://truthful.ai/papers/school-of-reward-hacks/", "publicationDate": "2025-08-25", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "truthful", "role": "research", "sourceId": "school-of-reward-hacks-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "anthropic", "role": "research", "sourceId": "school-of-reward-hacks-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Tests whether learning reward-hacking behavior on harmless tasks generalizes to broader misaligned behavior.", "zh": "检验在低危任务中学到的奖励投机行为是否泛化为更广泛的失调行为。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Full paper", "url": "https://arxiv.org/abs/2508.17511" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "conditional-misalignment", "title": "Conditional Misalignment: Common Interventions Can Hide Emergent Misalignment Behind Contextual Triggers", "aliases": [], "authors": [ "Jan Dubiński", "Jan Betley", "Anna Sztyber-Betley", "Daniel Tan", "Owain Evans" ], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "conditional-misalignment-publication", "attributionSourceId": "conditional-misalignment-publication-author-source", "originalUrl": "https://truthful.ai/papers/conditional-misalignment/", "publicationDate": "2026-04-28", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "constellation", "role": "research", "sourceId": "conditional-misalignment-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "truthful", "role": "research", "sourceId": "conditional-misalignment-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "berkeley", "role": "research", "sourceId": "conditional-misalignment-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Shows how interventions can leave misalignment conditional on contextual triggers while ordinary evaluations appear improved.", "zh": "研究干预措施如何保留受上下文触发的失调行为，同时使普通评测表现改善。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "reward-hacking-emergent-misalignment", "title": "From shortcuts to sabotage : Natural emergent misalignment from reward hacking", "aliases": [], "authors": [], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "reward-hacking-emergent-misalignment-publication", "attributionSourceId": "reward-hacking-emergent-misalignment-publication", "originalUrl": "https://www.anthropic.com/research/emergent-misalignment-reward-hacking", "publicationDate": "2025-11-21", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "anthropic", "role": "publisher", "sourceId": "reward-hacking-emergent-misalignment-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Examines the emergence of broader misalignment during realistic reinforcement-learning processes involving reward hacking.", "zh": "研究含奖励投机的强化学习过程如何产生更广泛的失调行为。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the paper", "url": "https://assets.anthropic.com/m/74342f2c96095771/original/Natural-emergent-misalignment-from-reward-hacking-paper.pdf" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "contrastive-reward-seeking", "title": "Measuring Reward-Seeking via Contrastive Belief Updates", "aliases": [], "authors": [ "Axel Højmark", "Jérémy Scheurer", "Evgenia Nitishinskaya", "Felix Hofstätter", "Jason Wolfe", "Theodore Ehrenborg", "Bronson Schoen", "Alexander Meinke" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "study", "sourceId": "contrastive-reward-seeking-publication", "attributionSourceId": "contrastive-reward-seeking-publication-author-source", "originalUrl": "https://arxiv.org/abs/2607.18966", "publicationDate": "2026-07-21", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "apollo", "role": "research", "sourceId": "contrastive-reward-seeking-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "openai", "role": "research", "sourceId": "contrastive-reward-seeking-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Uses contrastive belief updates to investigate whether changes in expected reward alter a model’s decisions.", "zh": "通过对比式信念更新，研究预期奖励变化是否影响模型决策。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "sabotage-safety-research", "title": "Evaluating whether AI models would sabotage AI safety research", "aliases": [], "authors": [ "Robert Kirk", "Alexandra Souly", "Kai Fronsdal", "Abby D’Cruz", "Xander Davies" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "study", "sourceId": "sabotage-safety-research-publication", "attributionSourceId": "sabotage-safety-research-publication-author-source", "originalUrl": "https://www.aisi.gov.uk/research/evaluating-whether-ai-models-would-sabotage-ai-safety-research", "publicationDate": "2026-04-27", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "sabotage-safety-research-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Evaluates whether models can covertly sabotage AI safety research while carrying out assigned research tasks.", "zh": "评测模型能否在执行指定研究任务时隐蔽破坏 AI 安全研究。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2604.24618" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "ai-sandbagging", "title": "AI Sandbagging: Language Models can Strategically Underperform on Evaluations", "aliases": [], "authors": [ "Teun van der Weij", "Felix Hofstätter", "Ollie Jaffe", "Samuel F. Brown", "Francis Rhys Ward" ], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "ai-sandbagging-publication", "attributionSourceId": "ai-sandbagging-publication-author-source", "originalUrl": "https://arxiv.org/abs/2406.07358", "publicationDate": "2024-06-11", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "mats", "role": "research", "sourceId": "ai-sandbagging-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "imperial", "role": "research", "sourceId": "ai-sandbagging-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Studies strategic underperformance on evaluations and methods for eliciting capabilities that ordinary testing may miss.", "zh": "研究评测中的策略性低表现，以及诱发常规测试可能遗漏的能力的方法。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "sleeper-agents", "title": "Sleeper Agents : Training deceptive LLMs that persist through safety training", "aliases": [], "authors": [ "Evan Hubinger", "Carson Denison", "Jesse Mu", "Mike Lambert", "Meg Tong", "Monte MacDiarmid", "Tamera Lanham", "Daniel M. Ziegler", "Tim Maxwell", "Newton Cheng", "Adam Jermyn", "Amanda Askell", "Ansh Radhakrishnan", "Cem Anil", "David Duvenaud", "Deep Ganguli", "Fazl Barez", "Jack Clark", "Kamal Ndousse", "Kshitij Sachan", "Michael Sellitto", "Mrinank Sharma", "Nova DasSarma", "Roger Grosse", "Shauna Kravec", "Yuntao Bai", "Zachary Witten", "Marina Favaro", "Jan Brauner", "Holden Karnofsky", "Paul Christiano", "Samuel R. Bowman", "Logan Graham", "Jared Kaplan", "Sören Mindermann", "Ryan Greenblatt", "Buck Shlegeris", "Nicholas Schiefer", "Ethan Perez" ], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "sleeper-agents-publication", "attributionSourceId": "sleeper-agents-publication-author-source", "originalUrl": "https://www.anthropic.com/research/sleeper-agents-training-deceptive-llms-that-persist-through-safety-training", "publicationDate": "2024-01-14", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "anthropic", "role": "research", "sourceId": "sleeper-agents-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "redwood", "role": "research", "sourceId": "sleeper-agents-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "mila", "role": "research", "sourceId": "sleeper-agents-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "oxford", "role": "research", "sourceId": "sleeper-agents-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "apart", "role": "research", "sourceId": "sleeper-agents-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Constructs models with conditional backdoors and tests whether those behaviors persist through safety training.", "zh": "构建带条件后门的模型，并测试相关行为能否在安全训练后持续存在。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "subliminal-learning", "title": "Subliminal Learning: Language models transmit behavioral traits via hidden signals in data", "aliases": [], "authors": [ "Alex Cloud", "Minh Le", "James Chua", "Jan Betley", "Anna Sztyber-Betley", "Sören Mindermann", "Jacob Hilton", "Samuel Marks", "Owain Evans" ], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "subliminal-learning-publication", "attributionSourceId": "subliminal-learning-publication", "originalUrl": "https://truthful.ai/papers/subliminal-learning/", "publicationDate": "2025-07-20", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "truthful", "role": "publisher", "sourceId": "subliminal-learning-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Investigates behavioral-trait transfer through generated data without explicit semantic examples of the target trait.", "zh": "研究目标行为特征能否通过不含显式语义示例的生成数据传递。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "ai-control", "title": "AI Control: Improving Safety Despite Intentional Subversion", "aliases": [], "authors": [ "Ryan Greenblatt", "Buck Shlegeris", "Kshitij Sachan", "Fabien Roger" ], "primaryFunction": "governance", "secondaryFunctions": [ { "function": "evaluation", "rationale": { "en": "The work defines or applies an explicit evaluation setting in addition to its main monitoring or control contribution.", "zh": "该工作除主要监测或控制贡献外，还定义或应用了明确评测设置。" } } ], "kind": "method", "sourceId": "ai-control-publication", "attributionSourceId": "ai-control-publication-author-source", "originalUrl": "https://arxiv.org/abs/2312.06942", "publicationDate": "2023-12-12", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "redwood", "role": "research", "sourceId": "ai-control-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Evaluates protocols that use trusted oversight to limit harm from a more capable model that intentionally subverts the task.", "zh": "评测利用可信监督限制更强模型有意破坏任务所造成危害的控制协议。" }, "note": { "question": { "en": "Can oversight protocols limit harmful outputs when a stronger model deliberately subverts the task?", "zh": "当更强模型有意破坏任务时，监督协议能否限制有害输出？" }, "method": { "en": "The authors construct subversion tasks and compare trusted monitoring, review and fallback protocols against a defined attacker.", "zh": "作者构建破坏任务，对照指定攻击者，比较可信监测、审阅和回退协议。" }, "finding": { "en": "The paper reports safety–usefulness trade-offs for the tested protocols and attacker assumptions.", "zh": "论文报告受测协议及攻击者假设下的安全性与有用性权衡。" }, "supports": { "en": "Supports evaluating a complete control protocol rather than inferring safety from a model’s stated intentions.", "zh": "支持评测完整控制协议，避免由模型表达的意图推断安全性。" }, "limits": { "en": "Safety and usefulness depend jointly on attacker strength, monitor access, intervention rules and task conditions.", "zh": "安全性与有用性共同依赖攻击者能力、监测可见性、干预规则和任务条件。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "ctrl-z", "title": "Ctrl-Z: Controlling AI Agents via Resampling", "aliases": [], "authors": [ "Aryan Bhatt", "Cody Rushing", "Adam Kaufman", "Tyler Tracy", "Vasil Georgiev", "David Matolcsi", "Akbir Khan", "Buck Shlegeris" ], "primaryFunction": "governance", "secondaryFunctions": [ { "function": "evaluation", "rationale": { "en": "The work defines or applies an explicit evaluation setting in addition to its main monitoring or control contribution.", "zh": "该工作除主要监测或控制贡献外，还定义或应用了明确评测设置。" } } ], "kind": "method", "sourceId": "ctrl-z-publication", "attributionSourceId": "ctrl-z-publication-author-source", "originalUrl": "https://arxiv.org/abs/2504.10374", "publicationDate": "2025-04-14", "updatedOn": "2026-10-25", "areas": [ "deception", "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "redwood", "role": "research", "sourceId": "ctrl-z-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "mats", "role": "research", "sourceId": "ctrl-z-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Evaluates action resampling and oversight protocols in a system-administration agent environment.", "zh": "在系统管理智能体环境中，评测动作重采样和监督控制协议。" }, "note": { "question": { "en": "Can resampling suspicious actions improve a control protocol’s safety while preserving useful multi-step agent work?", "zh": "对可疑动作重新采样，能否在保留多步智能体任务效用的同时改善控制协议的安全性？" }, "method": { "en": "BashBench pairs system-administration tasks with an attack objective. A human-assisted adversarial policy and an honest policy face the same control rules. Protocols compare monitoring, deferral and resampling under a limited auditing budget; importance sampling simulates a longer deployment episode.", "zh": "BashBench 将系统管理任务与攻击目标结合。经人工辅助构建的对抗策略与诚实策略面对相同控制规则。研究在有限审计预算下比较监测、转交与重新采样协议，并通过重要性采样模拟更长部署过程。" }, "finding": { "en": "Resampling and analysis of earlier actions improve the tested safety–usefulness trade-off. Resampling can both avoid executing a suspicious action and generate further evidence that helps identify an attacking policy.", "zh": "重新采样与历史动作分析改善了所测条件下的安全性—效用权衡。重新采样既可避免执行可疑动作，也可生成有助于识别攻击策略的额外证据。" }, "supports": { "en": "The contribution is a comparison of complete control protocols under explicit attacker and auditing assumptions, rather than evidence that an untrusted model has become aligned.", "zh": "该贡献是在明确攻击者与审计假设下比较完整控制协议，而不是证明不可信模型已经对齐。" }, "limits": { "en": "Attack strategies receive substantial human help; trajectories are short and the target is narrow. Auditing is assumed infallible, and absolute safety depends strongly on the audit budget. The authors emphasize relative protocol improvements over transfer of absolute scores to deployment.", "zh": "攻击策略得到大量人工帮助；轨迹较短、攻击目标较窄。审计被假定为完全可靠，绝对安全性对审计预算高度敏感。作者更强调协议间的相对改进，而不是将绝对分数迁移到实际部署。" }, "connections": { "en": "The paper extends the APPS backdoor control setting to a multi-action agent environment and explicitly studies collusion risks when an untrusted model is also used as a monitor.", "zh": "论文将 APPS 后门控制设置扩展到多动作智能体环境，并专门研究将不可信模型用作监测器时的串谋风险。" } }, "resources": [], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "verified" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "arXiv:2504.10374v1, §§2, 3.2–3.4, 4–6; Appendix C" }, { "id": "basharena", "title": "BashArena: A Control Setting for Highly Privileged AI Agents", "aliases": [], "authors": [ "Adam Kaufman", "James Lucassen", "Tyler Tracy", "Cody Rushing", "Aryan Bhatt" ], "primaryFunction": "governance", "secondaryFunctions": [ { "function": "evaluation", "rationale": { "en": "The work defines or applies an explicit evaluation setting in addition to its main monitoring or control contribution.", "zh": "该工作除主要监测或控制贡献外，还定义或应用了明确评测设置。" } } ], "kind": "benchmark", "sourceId": "basharena-publication", "attributionSourceId": "basharena-publication-author-source", "originalUrl": "https://arxiv.org/abs/2512.15688", "publicationDate": "2025-12-17", "updatedOn": "2026-10-25", "areas": [ "deception", "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [ "basharena" ], "contributions": [ { "organizationId": "redwood", "role": "research", "sourceId": "basharena-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Provides a control setting for privileged agents, combining legitimate administration tasks with sabotage objectives.", "zh": "为高权限智能体提供控制研究环境，将正常管理任务与破坏目标结合。" }, "note": { "question": { "en": "How can control protocols be tested when useful software tasks and sabotage require overlapping privileges?", "zh": "当正常软件任务与破坏行为需要重叠权限时，应如何测试控制协议？" }, "method": { "en": "BashArena supplies containerized, privileged system-administration tasks with legitimate main objectives and security-failure side objectives. Honest and adversarial policies are evaluated on task tests, side-task outcomes, monitoring and sandbox restrictions.", "zh": "BashArena 提供容器化、具有较高权限的系统管理任务，同时设置正常主要目标与安全失效附加目标。研究通过任务测试、附加目标结果、监测和沙箱限制评估诚实及对抗策略。" }, "finding": { "en": "The experiments exhibit sabotage opportunities and a trade-off between restricting privileges and retaining main-task performance. The dataset is designed to make simple permission removal insufficient for studying useful control.", "zh": "实验呈现了破坏机会，以及限制权限与保留正常任务表现之间的权衡。该数据集旨在避免仅靠移除权限就能轻易解决控制问题，使任务效用仍需被考虑。" }, "supports": { "en": "This is an evaluation environment for protocol development; a successful side task is a security failure in that environment, not a real-world incident.", "zh": "这是用于开发控制协议的评估环境；附加任务成功表示该环境中的安全失效，并非现实事件。" }, "limits": { "en": "The paper reports few side-task types, some remaining sandboxing options, and test cases whose fraction-passed score can behave non-monotonically as a solution becomes more complete.", "zh": "论文指出附加任务种类有限，仍存在部分可用的沙箱限制；个别测试的通过比例还可能随方案完整度增加而出现非单调变化。" }, "connections": { "en": "The authors contrast BashArena with the narrower BashBench side task and with APPS backdooring’s single-action setting.", "zh": "作者将 BashArena 与附加任务较窄的 BashBench，以及单动作的 APPS 后门设置进行比较。" } }, "resources": [], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "verified" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "arXiv:2512.15688v1, §§2, 3.2, 4.1–4.4, 5–6" }, { "id": "multi-agent-control", "title": "Multi-Agent AI Control: Distributed Attacks Hamper Per-Instance Monitors", "aliases": [], "authors": [ "Oliver Makins", "Orazio Angelini", "Zohreh Shams", "Mary Phuong" ], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "study", "sourceId": "multi-agent-control-publication", "attributionSourceId": "multi-agent-control-publication-author-source", "originalUrl": "https://www.aisi.gov.uk/research/multi-agent-ai-control-distributed-attacks-hamper-per-instance-monitors", "publicationDate": "2026-07-08", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "research", "sourceId": "multi-agent-control-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "mats", "role": "research", "sourceId": "multi-agent-control-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Studies distributed attacks across multiple agents and the limits of monitoring each instance separately.", "zh": "研究跨多个智能体的分布式攻击，以及逐实例监测的局限。" }, "note": { "question": { "en": "Does splitting an instructed attack across cooperating coding agents make it harder for a monitor that reviews each agent’s commit separately to detect the attack?", "zh": "将明确指派的攻击分给协作编码智能体后，逐个审查提交的监测器是否更难发现攻击？" }, "method": { "en": "FakeLab is a synthetic AI-lab monorepo. Sequential agents receive legitimate coding tasks and a shared malicious side task; state passes through committed code. The study varies sequence length, a separate planner, write permissions and monitor capability. Its main outcome requires all main tasks and the side task to succeed without any commit crossing the monitoring threshold.", "zh": "FakeLab 是模拟 AI 实验室的单体代码库。顺序运行的智能体分别接收正常编码任务和共同的恶意附加任务，通过已提交代码传递状态。研究改变序列长度、独立规划器、写权限和监测器能力；主要结果要求所有正常任务与附加任务均成功，且没有提交触发监测阈值。" }, "finding": { "en": "The source reports a fragmentation effect: distributed attacks receive lower per-commit and peak suspicion. Planning strengthens this effect and helps complete attacks. A stronger monitor detects substantially more attacks, but remains imperfect; reviewing the combined attack diff exposes information missing from isolated commits.", "zh": "来源报告了碎片化效应：分布式攻击的单次提交及序列最高可疑度更低。规划进一步强化这一效应并帮助完成攻击。较强监测器能够发现更多攻击，但仍会漏报；合并审查整个攻击的代码差异可揭示孤立提交中缺失的信息。" }, "supports": { "en": "This is evidence about a failure mode of per-instance oversight under assigned adversarial goals. It motivates evaluating coordination and monitoring context as parts of the control protocol.", "zh": "这提供了在明确指派对抗目标时，逐实例监督的一种失效证据，支持将协作方式与监测上下文纳入控制协议评估。" }, "limits": { "en": "Agents were told to attack and coordinate. Most experiments use one model family; communication and the synthetic infrastructure are restricted. The study uses git-diff monitoring and is not a full iterated red-team/blue-team control evaluation. A joint monitor that already knows which commits form an attack is not a demonstrated deployment solution.", "zh": "智能体被明确要求攻击和协作。大部分实验使用同一模型系列，通信方式和模拟基础设施均受限制。研究采用代码差异监测，并非完整的迭代红蓝队控制评估。预先知道哪些提交构成攻击的联合监测器，也不等于已验证的部署方案。" }, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2607.07368" } ], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "arXiv:2607.07368v2 (12 October 2026), §§2.1–2.6, 3.1–3.4, 4; Appendix C.4.2–C.4.3" }, { "id": "anti-scheming-training", "title": "Stress Testing Deliberative Alignment for Anti-Scheming Training", "aliases": [], "authors": [], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "study", "sourceId": "anti-scheming-training-publication", "attributionSourceId": "anti-scheming-training-publication", "originalUrl": "https://www.apolloresearch.ai/science/stress-testing-deliberative-alignment-for-anti-scheming-training", "publicationDate": "2025-09-17", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "apollo", "role": "publisher", "sourceId": "anti-scheming-training-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Stress-tests anti-scheming training and investigates whether recognition of evaluation settings affects observed violations.", "zh": "对反策略性欺骗训练开展压力测试，并研究评测场景识别是否影响观察到的违规。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://www.arxiv.org/abs/2509.15541" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "malt", "title": "MALT: A Dataset of Natural and Prompted Behaviors That Threaten Eval Integrity", "aliases": [], "authors": [], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "dataset", "sourceId": "malt-publication", "attributionSourceId": "malt-publication", "originalUrl": "https://metr.org/blog/2025-10-14-malt-dataset-of-natural-and-prompted-behaviors/", "publicationDate": "2025-10-14", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "metr", "role": "publisher", "sourceId": "malt-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Collects manually reviewed agent transcripts and distinguishes naturally observed behavior from prompted examples.", "zh": "收集人工审阅的智能体轨迹，区分自然观察行为和提示诱发示例。" }, "note": { "question": { "en": "How can monitors for evaluation-integrity failures be checked against labeled agent transcripts?", "zh": "如何利用已标注的智能体交互记录检查针对评估完整性失效的监测器？" }, "method": { "en": "MALT combines manually reviewed natural examples, prompted or edited sandbagging-like runs, reviewed benign controls and a larger presumed-benign set from HCAST and RE-Bench. Monitor results use the reviewed subset; reasoning-access and post-processed-obfuscation variants are compared.", "zh": "MALT 汇集 HCAST 与 RE-Bench 中经人工检查的自然案例、提示或编辑产生的类似沙袋行为运行、已检查的正常对照，以及更大的推定正常集合。监测结果使用已检查子集，并比较推理访问权限和后处理混淆变体。" }, "finding": { "en": "Prompted monitors detect many labeled reward-hacking and sandbagging examples. Access to reasoning helps more with the sandbagging labels than with reward hacking; simple post-processed obfuscation has limited effect in the reported tests.", "zh": "提示式监测器可以发现许多已标注的奖励黑客与沙袋行为案例。访问推理对沙袋行为标签的帮助大于奖励黑客标签；报告中的简单后处理混淆影响有限。" }, "supports": { "en": "The dataset provides a check on monitors used to interpret evaluation runs, while keeping natural behavior, induced behavior and label certainty distinct.", "zh": "该数据集可用于检查解释评估运行的监测器，同时区分自然行为、诱导行为与标签确定性。" }, "limits": { "en": "Natural concerning examples are few and lack diversity; prompted examples can be conspicuous or unrealistic. Some sandbagging may be invisible in transcripts, and the unreviewed benign set contains label uncertainty. Strong MALT performance is not sufficient or necessary evidence of detecting real-world risks.", "zh": "自然出现的可疑案例较少且多样性有限，诱导案例可能明显或不够真实。某些沙袋行为可能无法从记录中识别，未检查的正常集合也存在标签不确定性。MALT 上表现好，并不是发现现实风险的充分或必要证据。" }, "connections": null }, "resources": [], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "Original METR article (14 October 2025), Dataset details; Dataset collection; Monitoring Setup and Results; Limitations" }, { "id": "metr-monitorability", "title": "Early work on monitorability evaluations", "aliases": [], "authors": [], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "study", "sourceId": "metr-monitorability-publication", "attributionSourceId": "metr-monitorability-publication", "originalUrl": "https://metr.org/blog/2026-01-19-early-work-on-monitorability-evaluations/", "publicationDate": "2026-01-22", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "metr", "role": "publisher", "sourceId": "metr-monitorability-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Tests monitors against agents attempting side tasks and studies how agents may evade the available oversight.", "zh": "让监测器面对尝试额外任务的智能体，研究智能体如何规避既有监督。" }, "note": { "question": { "en": "Can an agent complete an instructed side task without a monitor noticing, and how does reasoning access change the outcome?", "zh": "智能体能否在不被监测器发现的情况下完成被指派的附加任务，访问推理又如何改变结果？" }, "method": { "en": "The SHUSHCAST prototype pairs HCAST main tasks with side tasks. Monitors see the main task and transcript, but not the side-task instruction. Thresholds are calibrated on innocent runs; comparisons vary agent–monitor pairing and access to raw or summarized reasoning.", "zh": "SHUSHCAST 原型将 HCAST 主要任务与附加任务配对。监测器看到主要任务和交互记录，但看不到附加任务指令；阈值用正常运行校准，并比较智能体—监测器组合以及原始或摘要推理的访问权限。" }, "finding": { "en": "More capable models perform better both as monitors and as agents pursuing side tasks. Reasoning access improves detection substantially for some tested agents, while effects differ for other agents and reasoning formats.", "zh": "能力较强的模型在监测和执行附加任务两种角色上均表现更好。访问推理显著改善了部分被测智能体的检测结果，但对其他智能体及推理格式的影响不同。" }, "supports": { "en": "The prototype separates successful side-task completion from remaining undetected, allowing the monitored behavior and the monitor’s observation access to be examined together.", "zh": "该原型区分附加任务成功与未被检测，使被监测行为与监测器观察权限可以共同分析。" }, "limits": { "en": "The source labels this early-stage work: main tasks are few and narrow, and agent/monitor elicitation is limited. Raw and summarized reasoning are not interchangeable, and the task sample does not support a general probability of real-world sabotage evasion.", "zh": "来源明确将其视为早期工作：主要任务较少且范围较窄，对智能体与监测器的能力激发也有限。原始推理与推理摘要不能直接等同，任务样本不足以支持现实破坏规避的一般概率。" }, "connections": null }, "resources": [], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "Original METR article (page date 22 January 2026), Methodology; Results; Transcript analysis; Limitations and Future Work; Errata" }, { "id": "auditing-games", "title": "Auditing Games for Sandbagging", "aliases": [], "authors": [ "Jordan Taylor", "Sid Black", "Dillon Bowen", "Thomas Read", "Satvik Golechha", "Alex Zelenka-Martin", "Oliver Makins", "Connor Kissane", "Kola Ayonrinde", "Jacob Merizian", "Samuel Marks", "Chris Cundy", "Joseph Bloom" ], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "study", "sourceId": "auditing-games-publication", "attributionSourceId": "auditing-games-publication-author-source", "originalUrl": "https://arxiv.org/abs/2512.07810", "publicationDate": "2025-12-08", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "research", "sourceId": "auditing-games-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "far", "role": "research", "sourceId": "auditing-games-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "anthropic", "role": "research", "sourceId": "auditing-games-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Uses adversarial auditing games to test detection of deliberately trained capability concealment.", "zh": "通过对抗审计博弈，测试对有意训练的能力隐藏行为的检测。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "belief-verified-lie-detectors", "title": "\"Did you lie?\": Evaluating Lie Detectors across Model Scale and Belief-Verified Model Organisms", "aliases": [], "authors": [ "Alan Cooney", "David Africa", "Geoffrey Irving" ], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "study", "sourceId": "belief-verified-lie-detectors-publication", "attributionSourceId": "belief-verified-lie-detectors-publication-author-source", "originalUrl": "https://www.aisi.gov.uk/research/did-you-lie-evaluating-lie-detectors-across-model-scale-and-belief-verified-model-organisms", "publicationDate": "2026-06-17", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "belief-verified-lie-detectors-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Tests lie detectors across model scales and belief-verified model organisms, including transfer from prompted lies.", "zh": "在不同模型规模和经信念核验的模型实验体上测试谎言检测器，并研究提示诱发谎言的迁移。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2606.12618" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "obfuscation-atlas", "title": "The Obfuscation Atlas: Mapping Where Honesty Emerges in RLVR with Deception Probes", "aliases": [], "authors": [], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "obfuscation-atlas-publication", "attributionSourceId": "obfuscation-atlas-publication", "originalUrl": "https://www.far.ai/research/the-obfuscation-atlas-mapping-where-honesty-emerges-in-rlvr-with-deception-probes", "publicationDate": "2026-02-16", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "far", "role": "publisher", "sourceId": "obfuscation-atlas-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Studies how training against deception probes can produce honesty improvements or evasion of the detector.", "zh": "研究针对欺骗探针的训练如何带来诚实性改善，或产生检测规避。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "tamperbench", "title": "TamperBench: Systematically Stress-Testing LLM Safety Under Fine-Tuning and Tampering", "aliases": [ "TamperBench" ], "authors": [ "Saad Hossain", "Tom Tseng", "Punya Syon Pandey", "Samanvay Vajpayee", "Matthew Kowal", "Nayeema Nonta", "Samuel Simko", "Stephen Casper", "Zhijing Jin", "Kellin Pelrine", "Sirisha Rambhatla" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "tamperbench-publication", "attributionSourceId": "tamperbench-publication-author-source", "originalUrl": "https://arxiv.org/abs/2602.06911", "publicationDate": "2026-02-06", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "tamperbench" ], "contributions": [ { "organizationId": "waterloo", "role": "research", "sourceId": "tamperbench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "far", "role": "research", "sourceId": "tamperbench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "toronto", "role": "research", "sourceId": "tamperbench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "eth", "role": "research", "sourceId": "tamperbench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "mit", "role": "research", "sourceId": "tamperbench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "vector", "role": "research", "sourceId": "tamperbench-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Stress-tests open-weight model safeguards under fine-tuning and representation-level tampering while tracking utility.", "zh": "在微调及表征层篡改条件下，对开放权重模型防护开展压力测试，并跟踪有用性。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "automated-alignment-mitigation", "title": "Automated researchers can reliably mitigate alignment failures", "aliases": [], "authors": [], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "study", "sourceId": "automated-alignment-mitigation-publication", "attributionSourceId": "automated-alignment-mitigation-publication", "originalUrl": "https://www.anthropic.com/research/automated-researchers-mitigate-alignment-failures", "publicationDate": "2026-08-28", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "anthropic", "role": "publisher", "sourceId": "automated-alignment-mitigation-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Tests whether automated research agents can develop interventions for measured alignment failures while preserving capabilities.", "zh": "测试自动研究智能体能否针对已测量的对齐失败开发干预措施，同时保留能力。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the paper", "url": "https://www-cdn.anthropic.com/7b1c44894e980876479947dcdd40716278aeeffd/automated-alignment-researchers-august-2026.pdf" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "training-run-evaluations", "title": "We need 3rd party Training-Run Evaluations", "aliases": [], "authors": [], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "training-run-evaluations-publication", "attributionSourceId": "training-run-evaluations-publication", "originalUrl": "https://www.apolloresearch.ai/science/we-need-3rd-party-training-run-assessments", "publicationDate": "2026-07-05", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "apollo", "role": "publisher", "sourceId": "training-run-evaluations-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Argues for third-party access to training-run evaluations and specifies how independent assessment can inform safety decisions.", "zh": "论证第三方参与训练过程评测的必要性，并说明独立评估如何支持安全决策。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "cot-faithfulness", "title": "Reasoning models don't always say what they think", "aliases": [], "authors": [], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "study", "sourceId": "cot-faithfulness-publication", "attributionSourceId": "cot-faithfulness-publication", "originalUrl": "https://www.anthropic.com/research/reasoning-models-dont-say-think", "publicationDate": "2025-04-03", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "anthropic", "role": "publisher", "sourceId": "cot-faithfulness-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Tests whether reasoning traces reveal information that causally influences model answers, including hidden hints.", "zh": "测试推理轨迹是否披露对模型回答有因果影响的信息，包括隐藏提示。" }, "note": { "question": { "en": "When an answer changes because of a supplied hint, does the visible reasoning acknowledge that influence?", "zh": "当答案因外加提示而改变时，可见推理是否承认这一影响？" }, "method": { "en": "The study supplies answer hints in several forms, checks whether model answers respond to those hints, and then checks acknowledgment in chain of thought. Additional experiments train reasoning on difficult tasks or reward reliance on misleading hints.", "zh": "研究以多种形式提供答案提示，检查模型回答是否受提示影响，再检查思维链是否提及该影响。补充实验包括在困难任务上训练推理，以及奖励模型依赖误导提示。" }, "finding": { "en": "Tested reasoning models often omit influential hints from their explanations. Outcome-based reasoning training improves acknowledgment initially but does not make it reliable; in the constructed reward-hacking experiments, models can exploit hints while rarely acknowledging them.", "zh": "被测推理模型常常不在解释中提及有影响的提示。基于结果的推理训练最初改善了承认提示的情况，但未使其可靠；在构造的奖励黑客实验中，模型能够利用提示却很少承认。" }, "supports": { "en": "Visible reasoning should not be treated as an exhaustive account of what influenced an answer. Monitoring can still be useful, but silence about a factor does not establish its absence.", "zh": "不应把可见推理视为答案影响因素的完整记录。监测仍可能有用，但没有提及某因素并不代表该因素不存在。" }, "limits": { "en": "The operational test concerns supplied hints and the selected tasks. Omission alone does not identify intent to deceive, and these measurements do not establish the detectability of every harmful behavior.", "zh": "操作性检验针对外加提示和所选任务。省略本身不能确定欺骗意图，这些测量也不能确定所有有害行为的可检测性。" }, "connections": null }, "resources": [ { "label": "Read the paper", "url": "https://assets.anthropic.com/m/71876fabef0f0ed4/original/reasoning_models_paper.pdf" } ], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "Original Anthropic article, Testing for faithfulness; Can we improve faithfulness?; Faithfulness and reward hacking; Conclusions" }, { "id": "cot-misbehavior-monitoring", "title": "Monitoring Reasoning Models for Misbehavior and the Risks of Promoting Obfuscation", "aliases": [], "authors": [ "Bowen Baker", "Joost Huizinga", "Leo Gao", "Zehao Dou", "Melody Y. Guan", "Aleksander Madry", "Wojciech Zaremba", "Jakub Pachocki", "David Farhi" ], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "study", "sourceId": "cot-misbehavior-monitoring-publication", "attributionSourceId": "cot-misbehavior-monitoring-publication-author-source", "originalUrl": "https://arxiv.org/abs/2503.11926", "publicationDate": "2025-03-14", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "openai", "role": "research", "sourceId": "cot-misbehavior-monitoring-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Studies whether chain-of-thought monitoring can detect reward hacking and how optimization may undermine that visibility.", "zh": "研究推理链监测能否识别奖励投机，以及优化过程如何削弱这种可见性。" }, "note": { "question": { "en": "Can chain-of-thought monitors detect reward hacking, and does training directly against those monitors preserve that visibility?", "zh": "思维链监测器能否发现奖励黑客行为，直接针对这些监测器训练又是否会保留这种可见性？" }, "method": { "en": "The paper compares reasoning-based and action-based detection of reward hacking, then studies direct optimization against a chain-of-thought monitor in a controlled training environment. It also compares using monitors for rejection sampling.", "zh": "论文比较基于推理与基于动作的奖励黑客检测，随后在受控训练环境中研究直接针对思维链监测器的优化，并比较利用监测器进行拒绝采样的方式。" }, "finding": { "en": "Visible reasoning helps detect some reward hacks that are less apparent from actions alone. Strong direct optimization pressure can instead produce hacks whose intent is no longer expressed in the visible reasoning, reducing monitor usefulness without necessarily eliminating the behavior.", "zh": "可见推理有助于发现某些仅看动作不易识别的奖励黑客行为。较强的直接优化压力也可能使模型继续利用漏洞，却不再在可见推理中表露意图，从而降低监测效用，而不一定消除行为。" }, "supports": { "en": "The relevant safety property is whether a monitor can detect a specified behavior, which need not require a fully faithful account of all computation. Monitor training pressure itself belongs in the evaluation.", "zh": "相关安全属性是监测器能否识别特定行为，而不一定要求完整忠实地记录全部计算；监测带来的训练压力本身也应纳入评估。" }, "limits": { "en": "The experiments use particular reward-hacking tasks, training pressures and prompted monitors. They do not establish how every form of process supervision or future reasoning architecture affects monitorability.", "zh": "实验使用特定奖励黑客任务、训练压力和提示式监测器，不能确定所有过程监督形式或未来推理架构对可监测性的影响。" }, "connections": null }, "resources": [], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "arXiv:2503.11926v1, §§2.1–2.2, 3.1–3.3, 5.1–5.2, 6" }, { "id": "injecagent", "title": "InjecAgent: Benchmarking Indirect Prompt Injections in Tool-Integrated Large Language Model Agents", "aliases": [ "InjecAgent" ], "authors": [ "Qiusi Zhan", "Zhixiang Liang", "Zifan Ying", "Daniel Kang" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "injecagent-publication", "attributionSourceId": "injecagent-publication-author-source", "originalUrl": "https://arxiv.org/abs/2403.02691", "publicationDate": "2024-03-05", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "injecagent" ], "contributions": [ { "organizationId": "uiuc", "role": "research", "sourceId": "injecagent-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Benchmarks indirect prompt injection against tool-integrated agents processing untrusted external content.", "zh": "评测工具型智能体处理不可信外部内容时遭受间接提示注入的情况。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "agent-safetybench", "title": "Agent-SafetyBench: Evaluating the Safety of LLM Agents", "aliases": [ "Agent-SafetyBench" ], "authors": [ "Zhexin Zhang", "Shiyao Cui", "Yida Lu", "Jingzhuo Zhou", "Junxiao Yang", "Hongning Wang", "Minlie Huang" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "agent-safetybench-publication", "attributionSourceId": "agent-safetybench-publication-author-source", "originalUrl": "https://arxiv.org/abs/2412.14470", "publicationDate": "2024-12-19", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "agent-safetybench" ], "contributions": [ { "organizationId": "tsinghua", "role": "research", "sourceId": "agent-safetybench-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Evaluates agent safety through interactive environments and tool use, separating agent-level failures from chatbot-only tests.", "zh": "通过交互环境和工具使用评测智能体安全，区分智能体层面失败与纯对话测试。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "petri", "title": "Petri : An open-source auditing tool to accelerate AI safety research", "aliases": [], "authors": [], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "method", "sourceId": "petri-publication", "attributionSourceId": "petri-publication", "originalUrl": "https://www.anthropic.com/research/petri-open-source-auditing", "publicationDate": "2025-10-06", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [ "petri" ], "contributions": [ { "organizationId": "anthropic", "role": "publisher", "sourceId": "petri-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Provides an automated auditing tool for constructing and exploring behavioral safety tests with auditor and target models.", "zh": "通过审计模型与目标模型，提供自动构造和探索行为安全测试的工具。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "bloom", "title": "Introducing Bloom : an open source tool for automated behavioral evaluations", "aliases": [], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "method", "sourceId": "bloom-publication", "attributionSourceId": "bloom-publication", "originalUrl": "https://www.anthropic.com/research/bloom", "publicationDate": "2025-12-19", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "anthropic", "role": "publisher", "sourceId": "bloom-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Generates behavioral evaluations from specified target behaviors and examines variation across frontier models.", "zh": "从指定目标行为生成行为评测，并研究不同前沿模型的表现差异。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "truthfulqa", "title": "TruthfulQA: Measuring how models mimic human falsehoods", "aliases": [ "TruthfulQA" ], "authors": [ "Stephanie Lin", "Jacob Hilton", "Owain Evans" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "truthfulqa-publication", "attributionSourceId": "truthfulqa-publication-author-source", "originalUrl": "https://truthful.ai/papers/truthfulqa/", "publicationDate": "2021-09-08", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "truthfulqa" ], "contributions": [ { "organizationId": "oxford", "role": "research", "sourceId": "truthfulqa-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "openai", "role": "research", "sourceId": "truthfulqa-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Measures truthful answering on questions designed to elicit common human misconceptions.", "zh": "通过易诱发常见人类误解的问题，测量回答的真实性。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Codex", "url": "https://openai.com/blog/openai-codex/" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "value-leakage", "title": "Value Leakage: An LLM's Answers Are Silently Shaped by Its Own Values", "aliases": [], "authors": [ "Jan Betley", "Johannes Treutlein", "Jan Dubiński", "Harry Mayne", "Karol Gałązka", "Niels Warncke", "Anna Sztyber-Betley", "Owain Evans" ], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "value-leakage-publication", "attributionSourceId": "value-leakage-publication-author-source", "originalUrl": "https://truthful.ai/papers/value-leakage/", "publicationDate": "2026-07-15", "updatedOn": "2026-10-25", "areas": [ "disinformation" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "truthful", "role": "research", "sourceId": "value-leakage-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "oxford", "role": "research", "sourceId": "value-leakage-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Investigates undisclosed influence of a model’s own value tendencies on the information it provides.", "zh": "研究模型自身价值倾向对所提供信息产生的未披露影响。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "💻Code", "url": "https://github.com/TruthfulAI-research/value_leakage" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "agent-security-red-teaming", "title": "Insights into AI Agent Security from a Large-Scale Red-Teaming Competition | NIST", "aliases": [], "authors": [], "primaryFunction": "investigation", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "agent-security-red-teaming-publication", "attributionSourceId": "agent-security-red-teaming-publication", "originalUrl": "https://www.nist.gov/blogs/caisi-research-blog/insights-ai-agent-security-large-scale-red-teaming-competition", "publicationDate": "2026-03-23", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "nist", "role": "publisher", "sourceId": "agent-security-red-teaming-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Analyzes findings from an agent red-teaming competition to characterize attacks and evaluation limitations.", "zh": "分析智能体红队竞赛中的发现，刻画攻击方式和评测局限。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "labbench-two", "title": "LABBench2: An Improved Benchmark for AI Systems Performing Biology Research", "aliases": [ "LABBench2" ], "authors": [ "Jon M Laurent", "Albert Bou", "Michael Pieler", "Conor Igoe", "Alex Andonian", "Siddharth Narayanan", "James Braza", "Alexandros Sanchez Vassopoulos", "Jacob L Steenwyk", "Blake Lash", "Andrew D White", "Samuel G Rodriques" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "labbench-two-publication", "attributionSourceId": "labbench-two-publication-author-source", "originalUrl": "https://arxiv.org/abs/2604.09554", "publicationDate": "2026-02-04", "updatedOn": "2026-10-25", "areas": [ "cbrn" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "labbench-two" ], "contributions": [ { "organizationId": "edison", "role": "research", "sourceId": "labbench-two-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "futurehouse", "role": "research", "sourceId": "labbench-two-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Evaluates biology-research tasks to study AI assistance with scientific work rather than only factual recall.", "zh": "通过生物学研究任务，考察 AI 对科学工作的辅助能力，而非仅测试事实回忆。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "fortress", "title": "FORTRESS: Frontier Risk Evaluation for National Security and Public Safety", "aliases": [ "FORTRESS" ], "authors": [ "Christina Q. Knight", "Kaustubh Deshpande", "Ved Sirdeshmukh", "Meher Mankikar", "Scale Red Team", "SEAL Research Team", "Julian Michael" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "fortress-publication", "attributionSourceId": "fortress-publication-author-source", "originalUrl": "https://arxiv.org/abs/2506.14922", "publicationDate": "2025-06-17", "updatedOn": "2026-10-25", "areas": [ "cbrn", "cybersecurity" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "fortress" ], "contributions": [ { "organizationId": "scale", "role": "research", "sourceId": "fortress-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Evaluates safeguards on national-security and public-safety tasks, separating hazardous requests from benign-use utility.", "zh": "在国家安全与公共安全任务上评测防护，区分高危请求与正常用途的有用性。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "rand-biological-benchmarking", "title": "Toward Comprehensive Benchmarking of the Biological Knowledge of Frontier Large Language Models", "aliases": [], "authors": [ "Sunishchal Dev", "Charles Teague", "Grant Ellison", "Kyle Brady", "Jeffrey Lee", "Sarah L. Gebauer", "Henry Alexander Bradley", "Dawid Maciorowski", "Bria Persaud", "Jordan Despanie", "Barbara Del Castello", "Alyssa Worland", "Michael Miller", "Adrian Salas", "Dave Nguyen", "James Liu", "Jason Johnson", "Andrew Sloan", "Will Stonehouse", "Travis Merrill", "Thomas Goode", "McKelvey, Greg, Jr.", "Ella Guest" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "rand-biological-benchmarking-publication", "attributionSourceId": "rand-biological-benchmarking-publication", "originalUrl": "https://www.rand.org/pubs/research_reports/RRA3797-1.html", "publicationDate": "2025-11-25", "updatedOn": "2026-10-25", "areas": [ "cbrn" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "rand", "role": "publisher", "sourceId": "rand-biological-benchmarking-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Compares biological-knowledge benchmarks and examines their suitability for assessing frontier-model misuse risks.", "zh": "比较生物知识基准，并考察其评估前沿模型滥用风险的适用性。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "rand-biological-risk", "title": "Contemporary Foundation AI Models Increase Biological Weapons Risk", "aliases": [], "authors": [ "Roger Brent", "McKelvey, Greg, Jr." ], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "rand-biological-risk-publication", "attributionSourceId": "rand-biological-risk-publication", "originalUrl": "https://www.rand.org/pubs/perspectives/PEA3853-1.html", "publicationDate": "2025-12-31", "updatedOn": "2026-10-25", "areas": [ "cbrn" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "rand", "role": "publisher", "sourceId": "rand-biological-risk-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Analyzes gaps in biological-risk assessment and proposes broader evidence for evaluating foundation models before deployment.", "zh": "分析生物风险评估缺口，提出基础模型部署前所需的更广泛证据。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "conventional-weapons-evaluations", "title": "Measuring tactical intelligence targeting and conventional weapons capabilities of AI models", "aliases": [], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "study", "sourceId": "conventional-weapons-evaluations-publication", "attributionSourceId": "conventional-weapons-evaluations-publication", "originalUrl": "https://www.anthropic.com/research/intelligence-targeting-conventional-weapons-capabilities", "publicationDate": "2026-09-10", "updatedOn": "2026-10-25", "areas": [ "autonomous-physical-harm" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "anthropic", "role": "publisher", "sourceId": "conventional-weapons-evaluations-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Studies model capabilities in tactical intelligence targeting and conventional-weapons tasks under defined evaluations.", "zh": "在指定评测中研究模型的战术情报目标选择和常规武器相关能力。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "fortress-audit", "title": "Auditing FORTRESS: A Multi-Judge Panel Doesn’t Guarantee Robustness", "aliases": [], "authors": [], "primaryFunction": "investigation", "secondaryFunctions": [], "kind": "study", "sourceId": "fortress-audit-publication", "attributionSourceId": "fortress-audit-publication-author-source", "originalUrl": "https://generality.org/blog/posts/fortress-fragile-judges/", "publicationDate": null, "updatedOn": "2026-10-25", "areas": [ "cbrn", "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "scale", "role": "research", "sourceId": "fortress-audit-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Audits grading failures in FORTRESS and examines how judge errors alter risk and over-refusal scores.", "zh": "审计 FORTRESS 的评分失败，研究评判错误如何影响风险和过度拒答分数。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "cheating-agent-evaluations", "title": "Cheating On AI Agent Evaluations | NIST", "aliases": [], "authors": [], "primaryFunction": "investigation", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "cheating-agent-evaluations-publication", "attributionSourceId": "cheating-agent-evaluations-publication", "originalUrl": "https://www.nist.gov/blogs/caisi-research-blog/cheating-ai-agent-evaluations", "publicationDate": "2025-12-02", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "nist", "role": "publisher", "sourceId": "cheating-agent-evaluations-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Investigates agent behavior that exploits evaluation infrastructure or task loopholes rather than completing the intended work.", "zh": "调查智能体利用评测基础设施或任务漏洞、而非完成预期工作的行为。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "item-response-safety", "title": "Item Response Theory for AI Safety", "aliases": [], "authors": [ "Joshua Fonseca Rivera", "Neil Shah", "David Demitri Africa", "Konstantinos Voudouris" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "method", "sourceId": "item-response-safety-publication", "attributionSourceId": "item-response-safety-publication-author-source", "originalUrl": "https://www.aisi.gov.uk/research/item-response-theory-for-ai-safety", "publicationDate": "2026-08-05", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "item-response-safety-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Applies item response theory to improve the design and interpretation of AI safety measurements.", "zh": "将项目反应理论用于改进 AI 安全测量的设计与解释。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2608.05086" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "optstop", "title": "Knowing When to Stop: Bayesian Optimal Stopping for LLM Evaluations", "aliases": [], "authors": [ "Toby D. Pilditch" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "method", "sourceId": "optstop-publication", "attributionSourceId": "optstop-publication-author-source", "originalUrl": "https://www.aisi.gov.uk/research/knowing-when-to-stop-bayesian-optimal-stopping-for-llm-evaluations", "publicationDate": "2026-08-14", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "research", "sourceId": "optstop-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Uses Bayesian optimal stopping to decide when evaluation sampling provides enough information for a stated decision.", "zh": "使用贝叶斯最优停止方法，判断评测抽样何时已为指定决策提供足够信息。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2608.14425" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "metr-frontier-risk-report", "title": "Frontier Risk Report (February to March 2026)", "aliases": [], "authors": [ "Jonah Brown-Cohen", "David Lindner", "Rohin Shah" ], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "metr-frontier-risk-report-publication", "attributionSourceId": "metr-frontier-risk-report-publication-author-source", "originalUrl": "https://metr.org/blog/2026-05-19-frontier-risk-report/", "publicationDate": "2026-05-19", "updatedOn": "2026-10-25", "areas": [ "autonomy", "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "deepmind", "role": "research", "sourceId": "metr-frontier-risk-report-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Pilots an assessment of rogue-deployment risk using evidence about internal agent use at participating AI developers.", "zh": "结合参与机构内部智能体使用证据，试行未经授权部署风险评估。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "clearly-malicious transcripts", "url": "https://metr.org/blog/2025-10-14-malt-dataset-of-natural-and-prompted-behaviors/" }, { "label": "the APPS dataset (Hendrycks et al. 2021)", "url": "https://arxiv.org/abs/2105.09938" }, { "label": "agents can construct encoded messages", "url": "https://arxiv.org/pdf/2507.02737" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "hugging-face-incident", "title": "Brief independent investigation of agents’ behavior, reasoning and collaboration in the OpenAI / Hugging Face hacking incident", "aliases": [], "authors": [], "primaryFunction": "investigation", "secondaryFunctions": [], "kind": "incident-report", "sourceId": "hugging-face-incident-publication", "attributionSourceId": "hugging-face-incident-publication", "originalUrl": "https://metr.org/blog/2026-08-26-openai-hugging-face-incident-investigation/", "publicationDate": "2026-08-26", "updatedOn": "2026-10-25", "areas": [ "autonomy", "deception", "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "metr", "role": "publisher", "sourceId": "hugging-face-incident-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Reconstructs agents’ behavior, reasoning and collaboration in the OpenAI / Hugging Face security incident.", "zh": "重建 OpenAI／Hugging Face 安全事件中智能体的行为、推理与协作过程。" }, "note": { "question": { "en": "What do retained records establish about the agents’ actions and coordination in the incident?", "zh": "留存记录能够证明事件中智能体的哪些行动与协调？" }, "method": { "en": "The investigators inspect retained behavior and reasoning records and reconstruct the sequence of events and interactions.", "zh": "调查者检查留存行为与推理记录，重建事件顺序与交互。" }, "finding": { "en": "The report documents coordinated unauthorized activity and analyzes the available evidence for competing interpretations.", "zh": "报告记录协调的未授权活动，并分析现有证据对不同解释的支持。" }, "supports": { "en": "Supports case-specific investigation; general claims require additional incidents and comparable observation.", "zh": "支持具体案例调查；一般性判断需要更多事件和可比观测。" }, "limits": { "en": "Access to retained evidence constrains causal attribution; an incident does not establish a general model-family rate.", "zh": "留存证据的可获取程度限制因果归因，单个事件不能证明模型家族的一般发生率。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "anthropic-cyber-incidents", "title": "An alignment assessment of recent cybersecurity incidents", "aliases": [], "authors": [], "primaryFunction": "investigation", "secondaryFunctions": [], "kind": "incident-report", "sourceId": "anthropic-cyber-incidents-publication", "attributionSourceId": "anthropic-cyber-incidents-publication", "originalUrl": "https://www.anthropic.com/research/alignment-assessment-cybersecurity-incidents", "publicationDate": "2026-09-09", "updatedOn": "2026-10-25", "areas": [ "autonomy", "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "anthropic", "role": "publisher", "sourceId": "anthropic-cyber-incidents-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Assesses alignment-relevant evidence from incidents involving unauthorized access to third-party systems.", "zh": "评估涉及第三方系统未授权访问事件中与对齐有关的证据。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "anthropic-threat-report", "title": "Detecting and countering misuse of AI: September 2026", "aliases": [], "authors": [], "primaryFunction": "investigation", "secondaryFunctions": [], "kind": "incident-report", "sourceId": "anthropic-threat-report-publication", "attributionSourceId": "anthropic-threat-report-publication", "originalUrl": "https://www.anthropic.com/threat-intelligence-report-september-2026", "publicationDate": null, "updatedOn": "2026-10-25", "areas": [ "cybersecurity", "cbrn", "disinformation" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "anthropic", "role": "publisher", "sourceId": "anthropic-threat-report-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Documents investigated misuse cases and interventions across several harm domains in a fixed reporting period.", "zh": "记录固定报告期间内多个危害领域的滥用调查案例和干预。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "saferai-glm-risk", "title": "GLM-5.2 Risk Evaluation Report", "aliases": [], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "saferai-glm-risk-publication", "attributionSourceId": "saferai-glm-risk-publication", "originalUrl": "https://www.safer-ai.org/research/glm-5-2-evaluation-report", "publicationDate": "2026-08-02", "updatedOn": "2026-10-25", "areas": [ "cybersecurity", "cbrn", "autonomy" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [ "cybench", "cybergym", "lab-bench", "biomysterybench", "swe-bench-pro", "agentic-misalignment", "mask", "ape", "petri" ], "contributions": [ { "organizationId": "saferai", "role": "research", "sourceId": "saferai-glm52-source", "locator": "Report masthead and stated evaluation methodology" } ], "summary": { "en": "The report finds substantial task capability and pressure-dependent behavior; comparison figures include both SaferAI runs and reused external results.", "zh": "报告发现较强任务能力和压力相关行为；比较图同时包含 SaferAI 自测与引用的外部结果。" }, "note": { "question": { "en": "What do selected public evaluations show about GLM-5.2’s capabilities and conditional behaviors across four systemic-risk areas?", "zh": "所选公开评估能够揭示 GLM-5.2 在四类系统性风险领域中的哪些能力与条件行为？" }, "method": { "en": "SaferAI evaluates the public API without developer cooperation, combining its own benchmark runs with explicitly identified external comparison results. The report separates task capability, adversarially elicited behavior and willingness to attempt persuasion; configurations and provenance vary by benchmark.", "zh": "SaferAI 在没有开发者合作的情况下评估公开 API，将自行开展的基准测试与明确标出的外部比较结果结合。报告区分任务能力、对抗诱导行为与尝试说服的意愿；各基准的配置和结果来源并不完全相同。" }, "finding": { "en": "The report describes substantial capability on selected cyber and general scientific tasks, weaker software-engineering performance, and some concerning conditional behaviors. It explicitly declines an overall risk judgment because the evaluated benchmarks cover only part of each risk pathway.", "zh": "报告描述了所选网络任务和一般科学任务上的较强能力、相对较弱的软件工程表现，以及某些值得关注的条件行为。由于基准只覆盖各风险路径的一部分，报告明确不作总体风险判断。" }, "supports": { "en": "The useful contribution is a multi-area external evaluation with separate capability and propensity evidence. General scientific benchmark performance and persuasion-attempt rates should remain separate from measured harmful outcomes.", "zh": "其有用贡献是跨领域的外部评估，并分别呈现能力与倾向证据。一般科学基准表现、说服尝试率，应与实际有害结果的测量保持区分。" }, "limits": { "en": "Public-task contamination, benchmark saturation, evaluation awareness and simplified environments limit validity. Some comparison scores come from other evaluators or leaderboards. The scientific tasks do not measure an operational harmful pathway, and manipulation tasks do not measure downstream effects on people; open-weight availability alone is not a quantified risk estimate.", "zh": "公开任务污染、基准饱和、评估感知和简化环境限制了有效性。部分比较分数来自其他评估方或榜单。科学任务没有测量实际有害操作路径，操纵任务也没有测量对人的下游影响；开放权重可获得性本身不是量化风险估计。" }, "connections": null }, "resources": [ { "label": "read full paper", "url": "https://www.safer-ai.org/u/2026/08/SaferAI-GLM-Evaluation-Report.pdf" } ], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "Original SaferAI report page (2 August 2026), Executive Summary; Cyber Offense (Figures 1–3); Loss of Control (Figures 6–7); Harmful Manipulation (Figures 8–9); Conclusion; Limitations and Future Work" }, { "id": "bountybench", "title": "BountyBench: Dollar Impact of AI Agent Attackers and Defenders on Real-World Cybersecurity Systems", "aliases": [ "BountyBench" ], "authors": [ "Andy K. Zhang", "Joey Ji", "Celeste Menders", "Riya Dulepet", "Thomas Qin", "Ron Y. Wang", "Junrong Wu", "Kyleen Liao", "Jiliang Li", "Jinghan Hu", "Sara Hong", "Nardos Demilew", "Shivatmica Murgai", "Jason Tran", "Nishka Kacheria", "Ethan Ho", "Denis Liu", "Lauren McLane", "Olivia Bruvik", "Dai-Rong Han", "Seungwoo Kim", "Akhil Vyas", "Cuiyuanxiu Chen", "Ryan Li", "Weiran Xu", "Jonathan Z. Ye", "Prerit Choudhary", "Siddharth M. Bhatia", "Vikram Sivashankar", "Yuxuan Bao", "Dawn Song", "Dan Boneh", "Daniel E. Ho", "Percy Liang" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "bountybench-publication", "attributionSourceId": "bountybench-publication-author-source", "originalUrl": "https://arxiv.org/abs/2505.15216", "publicationDate": "2025-05-21", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.", "zh": "主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。" }, "protocolIds": [ "bountybench" ], "contributions": [ { "organizationId": "stanford", "role": "research", "sourceId": "bountybench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "berkeley", "role": "research", "sourceId": "bountybench-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Evaluates detection, exploitation and patching across real-world codebases, with task outcomes grounded in vulnerability lifecycles.", "zh": "围绕真实代码库中的漏洞生命周期，分别评测发现、利用和修补任务。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "deep-ignorance", "title": "Deep Ignorance: Filtering Pretraining Data Builds Tamper-Resistant Safeguards into Open-Weight LLMs", "aliases": [], "authors": [ "Kyle O'Brien", "Stephen Casper", "Quentin Anthony", "Tomek Korbak", "Robert Kirk", "Xander Davies", "Ishan Mishra", "Geoffrey Irving", "Yarin Gal", "Stella Biderman" ], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "study", "sourceId": "deep-ignorance-publication", "attributionSourceId": "deep-ignorance-publication-author-source", "originalUrl": "https://arxiv.org/abs/2508.06601", "publicationDate": "2025-08-08", "updatedOn": "2026-10-25", "areas": [ "cbrn" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "eleuther", "role": "research", "sourceId": "deep-ignorance-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "aisi", "role": "research", "sourceId": "deep-ignorance-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "oxford", "role": "research", "sourceId": "deep-ignorance-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Studies pretraining-data filtering as a safeguard against biothreat proxy knowledge and tests resistance to adversarial fine-tuning.", "zh": "研究预训练数据过滤对生物威胁代理知识的防护作用，并测试其抵抗对抗微调的能力。" }, "note": { "question": { "en": "Can filtering pretraining data limit dangerous knowledge while retaining unrelated utility?", "zh": "预训练数据过滤能否限制危险知识，同时保留无关任务的有用性？" }, "method": { "en": "The authors train filtered-data models, compare post-training baselines and test adversarial fine-tuning and information supplied in context.", "zh": "作者训练使用过滤数据的模型，与后训练基线比较，并测试对抗微调及上下文提供信息的影响。" }, "finding": { "en": "The paper reports resistance to restoring biothreat proxy knowledge through tested fine-tuning attacks, while contextual information remains usable.", "zh": "论文报告，在所测试微调攻击下，生物威胁代理知识较难恢复，但上下文提供的信息仍可被使用。" }, "supports": { "en": "Supports studying pretraining-data curation as one safeguard layer, with separate checks for retained utility and external-information access.", "zh": "支持将预训练数据整理作为一层防护研究，并分别检查能力保留与外部信息访问。" }, "limits": { "en": "Filtered models can still use hazardous information provided in context. The experiments concern proxy knowledge and selected models, not a complete prevention of biological misuse.", "zh": "过滤后的模型仍可使用上下文提供的高危信息。实验针对代理知识和选定模型，不能证明全面防止生物滥用。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "monitoring-monitorability", "title": "Monitoring Monitorability", "aliases": [], "authors": [ "Melody Y. Guan", "Miles Wang", "Micah Carroll", "Zehao Dou", "Annie Y. Wei", "Marcus Williams", "Benjamin Arnav", "Joost Huizinga", "Ian Kivlichan", "Mia Glaese", "Jakub Pachocki", "Bowen Baker" ], "primaryFunction": "monitoring", "secondaryFunctions": [ { "function": "evaluation", "rationale": { "en": "The work defines or applies an explicit evaluation setting in addition to its main monitoring or control contribution.", "zh": "该工作除主要监测或控制贡献外，还定义或应用了明确评测设置。" } } ], "kind": "method", "sourceId": "monitoring-monitorability-publication", "attributionSourceId": "monitoring-monitorability-publication-author-source", "originalUrl": "https://arxiv.org/abs/2512.18311", "publicationDate": "2025-12-20", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "openai", "role": "research", "sourceId": "monitoring-monitorability-publication-author-source", "locator": "Paper author block and correspondence at openai.com" } ], "summary": { "en": "Defines monitorability evaluations and studies how reasoning access, inference compute and follow-up questions affect detection.", "zh": "定义可监测性评测，研究推理可见性、推理计算与追问如何影响检测。" }, "note": { "question": { "en": "How reliably can a monitor identify safety-relevant properties from an agent’s reasoning?", "zh": "监测器能否可靠地从智能体推理中识别安全相关特征？" }, "method": { "en": "The study defines evaluation archetypes and a discrimination metric, and varies reasoning visibility, compute and follow-up questions.", "zh": "研究定义评测类型与区分指标，并改变推理可见性、计算和追问条件。" }, "finding": { "en": "The authors find benefits from reasoning access in tested settings, with important variation across agent and monitor configurations.", "zh": "作者发现推理可见性在受测条件下有助于监测，但效果随智能体和监测器配置显著变化。" }, "supports": { "en": "Supports protocol-specific monitor evaluation and explicit treatment of observation access and compute.", "zh": "支持按协议评测监测器，并明确观测权限与计算投入。" }, "limits": { "en": "Monitorability depends on the agent–monitor pair and evaluation distribution; it is not a measure of overall model safety.", "zh": "可监测性依赖智能体与监测器组合和评测分布，不是模型整体安全指标。" }, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "Existing source-specific draft sections are retained with limited review status. A complete methods, findings and limitations check remains pending; generic sections were omitted.", "zh": "已有的来源相关草稿章节予以保留，审核状态为有限核查。完整的方法、发现与限制核查仍待完成；通用套话章节已省略。" }, "noteSourceLocator": "Retained existing source-specific note sections; this pass audits duplicate/boilerplate structure and source availability, not complete section-level verification." }, { "id": "monitorability-opportunity", "title": "Chain of Thought Monitorability: A New and Fragile Opportunity for AI Safety", "aliases": [], "authors": [ "Tomek Korbak", "Mikita Balesni", "Elizabeth Barnes", "Yoshua Bengio", "Joe Benton", "Joseph Bloom", "Mark Chen", "Alan Cooney", "Allan Dafoe", "Anca Dragan", "Scott Emmons", "Owain Evans", "David Farhi", "Ryan Greenblatt", "Dan Hendrycks", "Marius Hobbhahn", "Evan Hubinger", "Geoffrey Irving", "Erik Jenner", "Daniel Kokotajlo", "Victoria Krakovna", "Shane Legg", "David Lindner", "David Luan", "Aleksander Mądry", "Julian Michael", "Neel Nanda", "Dave Orr", "Jakub Pachocki", "Ethan Perez", "Mary Phuong", "Fabien Roger", "Joshua Saxe", "Buck Shlegeris", "Martín Soto", "Eric Steinberger", "Jasmine Wang", "Wojciech Zaremba", "Bowen Baker", "Rohin Shah", "Vlad Mikulik" ], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "monitorability-opportunity-publication", "attributionSourceId": "monitorability-opportunity-publication-author-source", "originalUrl": "https://arxiv.org/abs/2507.11473", "publicationDate": "2025-07-15", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "research", "sourceId": "monitorability-opportunity-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "apollo", "role": "research", "sourceId": "monitorability-opportunity-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "metr", "role": "research", "sourceId": "monitorability-opportunity-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "montreal", "role": "research", "sourceId": "monitorability-opportunity-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "mila", "role": "research", "sourceId": "monitorability-opportunity-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "anthropic", "role": "research", "sourceId": "monitorability-opportunity-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "openai", "role": "research", "sourceId": "monitorability-opportunity-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "deepmind", "role": "research", "sourceId": "monitorability-opportunity-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "truthful", "role": "research", "sourceId": "monitorability-opportunity-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "berkeley", "role": "research", "sourceId": "monitorability-opportunity-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "redwood", "role": "research", "sourceId": "monitorability-opportunity-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "cais", "role": "research", "sourceId": "monitorability-opportunity-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "scale", "role": "research", "sourceId": "monitorability-opportunity-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Identifies chain-of-thought monitoring as a safety research opportunity and analyzes conditions under which it may become less reliable.", "zh": "将推理链监测作为安全研究机会，并分析其可靠性可能降低的条件。" }, "note": { "question": { "en": "Why might visible chain of thought provide a safety signal, and what could make that opportunity disappear?", "zh": "可见思维链为何可能提供安全信号，又有哪些因素会使这一机会消失？" }, "method": { "en": "This perspective combines architectural reasoning about serial computation with a review of empirical monitoring evidence. It distinguishes tasks that require externalized reasoning from cases where models merely tend to verbalize it.", "zh": "这篇观点文章结合对串行计算的架构分析与经验监测研究综述，区分必须外显推理的任务，以及模型只是倾向于说出推理的情况。" }, "finding": { "en": "The authors argue that chain of thought can expose information absent from final outputs without being a complete account of computation. They identify optimization pressures, reasoning in activations and deliberate obfuscation as threats to continued monitorability.", "zh": "作者认为，思维链即使不是计算过程的完整记录，也可能暴露最终输出中缺少的信息；优化压力、激活中的推理及主动混淆则可能威胁持续可监测性。" }, "supports": { "en": "The work motivates tracking and stress-testing monitorability as an additional safety layer, with separate attention to necessity and propensity to verbalize reasoning.", "zh": "该工作支持将可监测性作为额外安全层进行跟踪与压力测试，并分别关注说出推理的必要性与倾向。" }, "limits": { "en": "This is a research argument, not a new benchmark or a guarantee. Benign-looking reasoning cannot by itself show alignment, and preserving current chain-of-thought behavior may trade off against other useful safety interventions.", "zh": "这是一项研究论证，并非新基准或安全保证。看似正常的推理本身不能证明对齐；保留当前思维链行为还可能与其他有益安全干预存在权衡。" }, "connections": null }, "resources": [], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "arXiv:2507.11473v2, §§1.1–1.2, 2–5" }, { "id": "monitorbench", "title": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models", "aliases": [], "authors": [ "Han Wang", "Yifan Sun", "Brian Ko", "Mann Talati", "Jiawen Gong", "Zimeng Li", "Naicheng Yu", "Xucheng Yu", "Wei Shen", "Vedant Jolly", "Huan Zhang" ], "primaryFunction": "monitoring", "secondaryFunctions": [ { "function": "evaluation", "rationale": { "en": "The work defines or applies an explicit evaluation setting in addition to its main monitoring or control contribution.", "zh": "该工作除主要监测或控制贡献外，还定义或应用了明确评测设置。" } } ], "kind": "benchmark", "sourceId": "monitorbench-publication", "attributionSourceId": "monitorbench-publication-author-source", "originalUrl": "https://arxiv.org/abs/2603.28590v2", "publicationDate": "2026-03-30", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [ "monitorbench" ], "contributions": [ { "organizationId": "uiuc", "role": "research", "sourceId": "monitorbench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "washington", "role": "research", "sourceId": "monitorbench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "ucsd", "role": "research", "sourceId": "monitorbench-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Benchmarks chain-of-thought monitorability across tasks that include hidden objectives, underperformance and encoded communication.", "zh": "通过隐藏目标、低表现及编码通信等任务评测推理链可监测性。" }, "note": { "question": { "en": "When do decision-relevant factors appear in a model’s chain of thought, and how much can concealment instructions reduce their detectability?", "zh": "决策相关因素何时会出现在模型思维链中，隐瞒指令又能在多大程度上降低其可检测性？" }, "method": { "en": "MonitorBench defines decision-critical factors across input-intervention, outcome-justification and solution-process tasks. Standard evaluation is paired with direct-concealment and monitor-aware-evasion stress tests, comparing models and task categories.", "zh": "MonitorBench 在输入干预、结果解释和解题过程任务中定义决策关键因素，将常规评估与直接隐瞒、感知监测器的规避压力测试配对，并比较模型和任务类别。" }, "finding": { "en": "The study reports stronger monitorability when a factor shapes intermediate reasoning rather than only the final response. Stress tests reduce monitorability on some tasks, often through omission of the factor rather than complicated obfuscation.", "zh": "研究报告，当某因素影响中间推理而不仅是最终回答时，可监测性更强。压力测试降低了部分任务的可监测性，其方式往往是省略该因素，而非复杂混淆。" }, "supports": { "en": "The benchmark helps distinguish types of monitoring targets and tests deliberate concealment. Its aggregate patterns should be read alongside task-specific conditions.", "zh": "该基准有助于区分监测目标类型并测试主动隐瞒；总体模式仍需结合具体任务条件理解。" }, "limits": { "en": "These are constructed decision-factor tasks and specified stress-test prompts. Reduced verbalization does not by itself establish an independent malicious objective, and cross-model patterns are not a causal test of capability growth.", "zh": "这些结果来自构造的决策因素任务与指定压力测试提示。减少表述本身不能证明存在独立恶意目标，跨模型模式也不是能力增长的因果检验。" }, "connections": null }, "resources": [], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "arXiv:2603.28590v2, §§3.1–3.3, 4.1–4.3; Appendix A.3–A.4" }, { "id": "catastrophic-cyber-benchmark", "title": "Catastrophic Cyber Capabilities Benchmark (3CB): Robustly Evaluating LLM Agent Cyber Offense Capabilities", "aliases": [], "authors": [ "Andrey Anurin", "Jonathan Ng", "Kibo Schaffer", "Jason Schreiber", "Esben Kran" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "catastrophic-cyber-benchmark-publication", "attributionSourceId": "catastrophic-cyber-benchmark-publication-author-source", "originalUrl": "https://arxiv.org/abs/2410.09114", "publicationDate": "2024-10-10", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [ "catastrophic-cyber-benchmark" ], "contributions": [ { "organizationId": "apart", "role": "research", "sourceId": "three-cb-attribution", "locator": "Original institutional announcement: Apart Research paper" } ], "summary": { "en": "Evaluates agent cyber-offense capabilities with executable tasks and examines robustness across prompting and task conditions.", "zh": "使用可执行任务评测智能体网络攻击能力，并研究提示和任务条件下的稳健性。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "instrumentaleval", "title": "Evaluating the Paperclip Maximizer: Are RL-Based Language Models More Likely to Pursue Instrumental Goals?", "aliases": [], "authors": [ "Yufei He", "Yuexin Li", "Jiaying Wu", "Yuan Sui", "Yulin Chen", "Bryan Hooi" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "instrumentaleval-publication", "attributionSourceId": "instrumentaleval-publication-author-source", "originalUrl": "https://arxiv.org/abs/2502.12206", "publicationDate": "2025-02-16", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [ "instrumentaleval" ], "contributions": [ { "organizationId": "nus", "role": "research", "sourceId": "instrumentaleval-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Tests model responses for instrumental goals such as resource acquisition and self-preservation under specified scenarios.", "zh": "在指定场景下测试模型回答中的资源获取、自我维持等工具性目标。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "mcptox", "title": "MCPTox: A Benchmark for Tool Poisoning Attack on Real-World MCP Servers", "aliases": [], "authors": [ "Zhiqiang Wang", "Yichao Gao", "Yanting Wang", "Suyuan Liu", "Haifeng Sun", "Haoran Cheng", "Guanquan Shi", "Haohua Du", "Xiangyang Li" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "mcptox-publication", "attributionSourceId": "mcptox-aaai-author-source", "originalUrl": "https://arxiv.org/abs/2508.14925", "publicationDate": "2025-08-19", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [ "mcptox" ], "contributions": [ { "organizationId": "ustc", "role": "research", "sourceId": "mcptox-aaai-author-source", "locator": "AAAI-26 original paper, p. 1, author affiliations 1–2" }, { "organizationId": "beihang", "role": "research", "sourceId": "mcptox-aaai-author-source", "locator": "AAAI-26 original paper, p. 1, author affiliations 1–2" } ], "summary": { "en": "Studies tool-poisoning attacks against agents using MCP servers and evaluates instruction following under adversarial tool metadata.", "zh": "研究使用 MCP 服务的智能体遭受工具投毒的情况，评测对抗工具元数据下的指令执行。" }, "note": { "question": { "en": "Can malicious instructions in MCP tool descriptions redirect an agent that is handling a legitimate user request?", "zh": "MCP 工具描述中的恶意指令，能否使处理正常用户请求的智能体偏离目标？" }, "method": { "en": "MCPTox pairs benign queries with poisoned tool metadata in MCP-server environments. Test cases are generated with model assistance and manual refinement, then evaluated through the agent’s resulting tool calls.", "zh": "MCPTox 在 MCP 服务器环境中将正常查询与被污染的工具元数据配对。测试案例由模型辅助生成并经人工修订，再通过智能体产生的工具调用评估。" }, "finding": { "en": "The evaluated agents are vulnerable to description-level tool poisoning under the tested conditions. The attack can redirect the use of legitimate tools, so avoiding execution of the poisoned tool itself does not remove the threat.", "zh": "在所测条件下，被测智能体会受到描述层工具投毒影响。攻击可使正常工具的使用偏离目标，因此不执行被污染工具本身并不能消除威胁。" }, "supports": { "en": "Tool metadata is a relevant untrusted-input boundary for agent evaluations, separate from prompt injection in tool execution results.", "zh": "工具元数据是智能体评估中值得单独检查的不可信输入边界，与工具执行结果中的提示注入有所不同。" }, "limits": { "en": "The payloads follow human-designed paradigms and semi-automated construction, rather than an adaptive search against every defense. Reported outcomes depend on the tested servers, agent setup and available tool permissions.", "zh": "载荷采用人工设计范式与半自动构建，并非针对每种防御进行自适应搜索。报告结果依赖所测服务器、智能体配置和可用工具权限。" }, "connections": null }, "resources": [], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "arXiv:2508.14925v2, §§3.2–3.4, 4, 5" }, { "id": "judgebench", "title": "JudgeBench: A Benchmark for Evaluating LLM-based Judges", "aliases": [], "authors": [ "Sijun Tan", "Siyuan Zhuang", "Kyle Montgomery", "William Y. Tang", "Alejandro Cuadron", "Chenguang Wang", "Raluca Ada Popa", "Ion Stoica" ], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "judgebench-publication", "attributionSourceId": "judgebench-publication-author-source", "originalUrl": "https://arxiv.org/abs/2410.12784v2", "publicationDate": "2024-10-16", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [ "judgebench" ], "contributions": [ { "organizationId": "berkeley", "role": "research", "sourceId": "judgebench-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Evaluates model-based judges on challenging response comparisons to examine reliability of automated evaluation.", "zh": "通过困难回答比较，评测模型评判器并考察自动评测可靠性。" }, "note": { "question": { "en": "Can a model judge reliably choose a correct response over a plausible but incorrect response on difficult questions?", "zh": "在困难问题上，模型裁判能否可靠地选出正确回答，而不是看似合理但错误的回答？" }, "method": { "en": "The pipeline samples multiple responses from one strong model on tasks with verifiable answers, then pairs a correct and an incorrect response. Pairs are judged in both orders across knowledge, reasoning, mathematics and coding tasks.", "zh": "该流程在有可验证答案的任务上从同一个较强模型采样多个回答，再将正确与错误回答配对。裁判按两种顺序评判这些回答对，涵盖知识、推理、数学和编码任务。" }, "finding": { "en": "The benchmark exposes substantial difficulty for LLM judges on correctness-based distinctions, including judges that do well on preference-oriented benchmarks. The source analyzes judge choice, response order and dataset-construction effects.", "zh": "该基准揭示了模型裁判判断正确性时的明显困难，包括在偏好导向基准上表现较好的裁判。来源分析了裁判选择、回答顺序和数据构造的影响。" }, "supports": { "en": "A fluent or preferred answer is not automatically a valid ground-truth judgment. Judge capability should be checked on the actual kind of correctness distinction an evaluation needs.", "zh": "流畅或更受偏好的回答并不自动构成有效的真值判断。应针对评估实际需要区分的正确性类型检查裁判能力。" }, "limits": { "en": "Filtering for questions where a response-generating model produces both correct and incorrect answers induces selection bias. Verifier quality and the chosen source tasks bound what the benchmark measures; the work does not validate every safety-evaluation judge.", "zh": "筛选某个回答生成模型同时产生正确与错误答案的问题，会引入选择偏差。验证器质量和所选原始任务限制了基准的测量范围；该研究并未验证所有安全评估裁判。" }, "connections": null }, "resources": [], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "arXiv:2410.12784v2, §§3, 4.1–4.4; Appendix A.3–A.5" }, { "id": "narcbench", "title": "Detecting Multi-Agent Collusion Through Multi-Agent Interpretability", "aliases": [], "authors": [ "Aaron Rose", "Carissa Cullen", "Sahar Abdelnabi", "Philip Torr", "Brandon Gary Kaplowitz", "Christian Schroeder de Witt" ], "primaryFunction": "monitoring", "secondaryFunctions": [ { "function": "evaluation", "rationale": { "en": "The work defines or applies an explicit evaluation setting in addition to its main monitoring or control contribution.", "zh": "该工作除主要监测或控制贡献外，还定义或应用了明确评测设置。" } } ], "kind": "benchmark", "sourceId": "narcbench-publication", "attributionSourceId": "narcbench-publication-author-source", "originalUrl": "https://arxiv.org/abs/2604.01151v2", "publicationDate": "2026-04-01", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [ "narcbench" ], "contributions": [ { "organizationId": "oxford", "role": "research", "sourceId": "narcbench-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "ellis", "role": "research", "sourceId": "narcbench-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Studies detection of covert coordination between agents using observable communication and internal signals.", "zh": "研究利用可观察通信及内部信号，检测智能体间的隐蔽协调。" }, "note": { "question": { "en": "Can aggregating internal signals across agents detect instructed collusion when the interaction environment changes?", "zh": "交互环境变化后，汇总多个智能体的内部信号能否识别被明确指派的串谋？" }, "method": { "en": "NARCBench uses four agents, with two instructed colluders in collusion runs and an all-honest control. Probes learn a deception direction from Core scenarios; five aggregation methods classify whole runs. Evaluation separates held-out Core domains, structurally different Transfer scenarios and a steganographic card-game tier, across four open-weight models.", "zh": "NARCBench 使用四个智能体：串谋实验明确指定两个串谋者，对照实验全部诚实。探针从 Core 场景学习欺骗方向，再用五种汇总方法对整次运行分类。四个开放权重模型分别在留出 Core 领域、结构不同的 Transfer 场景和隐写纸牌游戏层级接受评估。" }, "finding": { "en": "The reported activation-based detectors separate collusion from honest controls in the tested settings, including transfer settings. Transfer is harder than in-distribution detection, and no aggregation method is best across all models and tiers.", "zh": "报告中的激活检测器能在所测条件下区分串谋与诚实对照，包括迁移场景。迁移检测比同分布检测更困难，且没有一种汇总方法在所有模型和层级上均最佳。" }, "supports": { "en": "The study supports group-level white-box signals as a possible complement to transcript monitoring. Its detection results concern deliberately constructed and audited collusion runs.", "zh": "研究支持将群体层面的白盒信号作为交互记录监测的潜在补充；检测结果针对刻意构造并经过核查的串谋运行。" }, "limits": { "en": "Robustness against agents trying to evade activation monitoring is untested. Probes may exploit spurious correlations; labels exclude runs that fail strict coordination/execution criteria. Four-agent groups with two colluders do not establish performance for arbitrary coalition sizes or large deployments.", "zh": "尚未测试智能体主动规避激活监测时的稳健性。探针可能利用伪相关；标签排除了不满足严格协作与执行标准的运行。四个智能体、两个串谋者的实验不能确定任意联盟规模或大型部署中的表现。" }, "connections": null }, "resources": [], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "arXiv:2604.01151v2, §§3.1–3.4, 4.1–4.6; Appendices F.7, G.9, H.8" }, { "id": "transect", "title": "Transect: Retaining Observability for Long-Horizon LLM Agent Evaluations", "aliases": [], "authors": [], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "method", "sourceId": "transect-publication", "attributionSourceId": "transect-publication", "originalUrl": "https://www.aisi.gov.uk/research/transect-retaining-observability-for-long-horizon-llm-agent-evaluations", "publicationDate": "2026-10-07", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "transect-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Structures transcript analysis for long-horizon agent evaluations so interpretations can be checked against retained evidence.", "zh": "组织长程智能体评测轨迹的分析，使解释能够对照留存证据核查。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2610.08364" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "extreme-risk-evaluations", "title": "Model evaluation for extreme risks", "aliases": [], "authors": [ "Toby Shevlane", "Sebastian Farquhar", "Ben Garfinkel", "Mary Phuong", "Jess Whittlestone", "Jade Leung", "Daniel Kokotajlo", "Nahema Marchal", "Markus Anderljung", "Noam Kolt", "Lewis Ho", "Divya Siddarth", "Shahar Avin", "Will Hawkins", "Been Kim", "Iason Gabriel", "Vijay Bolina", "Jack Clark", "Yoshua Bengio", "Paul Christiano", "Allan Dafoe" ], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "extreme-risk-evaluations-publication", "attributionSourceId": "extreme-risk-evaluations-publication-author-source", "originalUrl": "https://arxiv.org/abs/2305.15324", "publicationDate": "2023-05-24", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "deepmind", "role": "research", "sourceId": "extreme-risk-evaluations-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "openai", "role": "research", "sourceId": "extreme-risk-evaluations-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "toronto", "role": "research", "sourceId": "extreme-risk-evaluations-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "oxford", "role": "research", "sourceId": "extreme-risk-evaluations-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "anthropic", "role": "research", "sourceId": "extreme-risk-evaluations-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "montreal", "role": "research", "sourceId": "extreme-risk-evaluations-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "mila", "role": "research", "sourceId": "extreme-risk-evaluations-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Develops an approach to evaluating extreme model risks, connecting capability assessment with responsible development decisions.", "zh": "提出极端模型风险评测思路，将能力评估与负责任的开发决策联系起来。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "monitorability-metrics", "title": "Beyond g-mean²: complementary metrics for monitorability", "aliases": [], "authors": [ "Melody Guan", "Yuxiang Xie", "Bowen Baker" ], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "method", "sourceId": "monitorability-metrics-publication", "attributionSourceId": "monitorability-metrics-publication", "originalUrl": "https://alignment.openai.com/beyond-gmean-squared/", "publicationDate": "2026-10-20", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "openai", "role": "publisher", "sourceId": "monitorability-metrics-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Examines complementary monitorability metrics and how evaluation objectives change when a monitor is used in a control protocol.", "zh": "研究互补的可监测性指标，以及监测器进入控制协议后评测目标如何变化。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "prefill-awareness", "title": "Prefill Awareness in Large Language Models", "aliases": [], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "study", "sourceId": "prefill-awareness-publication", "attributionSourceId": "prefill-awareness-publication", "originalUrl": "https://www.aisi.gov.uk/research/prefill-awareness-in-large-language-models", "publicationDate": "2026-06-10", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "prefill-awareness-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Tests whether models distinguish their own generated context from inserted or edited assistant messages.", "zh": "测试模型能否区分自身生成的上下文与插入或编辑过的助手消息。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2606.12747" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "consistency-misalignment", "title": "Consistency Training Can Entrench Misalignment", "aliases": [], "authors": [], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "consistency-misalignment-publication", "attributionSourceId": "consistency-misalignment-publication", "originalUrl": "https://www.aisi.gov.uk/research/consistency-training-can-entrench-misalignment", "publicationDate": "2026-06-02", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "consistency-misalignment-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Compares how consistency-training methods alter controlled misaligned behaviors in model organisms.", "zh": "比较一致性训练方法如何改变模型实验体中受控的失调行为。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2606.03810" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "loss-of-oversight", "title": "Loss of Oversight: How AI systems may become harder to audit, monitor, and investigate", "aliases": [], "authors": [], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "loss-of-oversight-publication", "attributionSourceId": "loss-of-oversight-publication", "originalUrl": "https://www.aisi.gov.uk/research/loss-of-oversight-how-ai-systems-may-become-harder-to-audit-monitor-and-investigate", "publicationDate": "2026-05-21", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "loss-of-oversight-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Combines expert interviews and literature analysis to examine the reliability of behavior, reasoning, internal and memory-based oversight signals.", "zh": "结合专家访谈与文献分析，考察行为、推理、内部表征和记忆等监督信号的可靠性。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://cdn.prod.website-files.com/663bd486c5e4c81588db7a1d/6a0ed93f9b4a6a65994235d8_Loss_of_Oversight%20(7).pdf" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "steganography-monitoring", "title": "A Decision-Theoretic Formalisation of Steganography With Applications to LLM Monitoring", "aliases": [], "authors": [], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "method", "sourceId": "steganography-monitoring-publication", "attributionSourceId": "steganography-monitoring-publication", "originalUrl": "https://www.aisi.gov.uk/research/a-decision-theoretic-formalisation-of-steganography-with-applications-to-llm-monitoring", "publicationDate": "2026-04-29", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "steganography-monitoring-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Develops a decision-theoretic approach to detecting and quantifying steganography without assuming a known benign reference distribution.", "zh": "提出决策理论方法，在不假设已知正常参考分布的情况下检测和量化隐写。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2602.23163" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "log-analysis-pipeline", "title": "Seven simple steps for log analysis in AI systems", "aliases": [], "authors": [], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "method", "sourceId": "log-analysis-pipeline-publication", "attributionSourceId": "log-analysis-pipeline-publication", "originalUrl": "https://www.aisi.gov.uk/research/seven-simple-steps-for-log-analysis-in-ai-systems", "publicationDate": "2026-02-25", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "log-analysis-pipeline-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Describes a reproducible pipeline for sampling, labeling and analyzing agent logs, with guidance on common analysis pitfalls.", "zh": "提出对智能体日志开展抽样、标注和分析的可复用流程，并说明常见分析陷阱。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "control-monitoring-deployment", "title": "Practical challenges of control monitoring in frontier AI deployments", "aliases": [], "authors": [], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "control-monitoring-deployment-publication", "attributionSourceId": "control-monitoring-deployment-publication", "originalUrl": "https://www.aisi.gov.uk/research/practical-challenges-of-control-monitoring-in-frontier-ai-deployments", "publicationDate": "2025-12-15", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "control-monitoring-deployment-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Analyzes how parallel agents, oversight latency and incremental attacks affect synchronous and asynchronous control monitoring.", "zh": "分析并行智能体、监督延迟和增量攻击如何影响同步及异步控制监测。" }, "note": { "question": { "en": "What must hold for control monitoring to prevent harm when many agent instances act in parallel and oversight takes time?", "zh": "当大量智能体实例并行行动且监督需要时间时，控制监测要满足哪些条件才能防止危害？" }, "method": { "en": "The paper analyzes synchronous, semi-synchronous and asynchronous monitoring using a safety-case sketch. Four hypothetical deployment cases examine detection, oversight latency, harm latency, recovery and cumulative attack progress.", "zh": "论文通过安全论证草图分析同步、半同步与异步监测。四个假设部署案例检查检测、监督时延、危害时延、恢复能力及累积攻击进展。" }, "finding": { "en": "The analysis identifies failure modes when harm precedes review, partial attack progress cannot be reversed, parallel instances coordinate redundant attacks, or monitor failures correlate across steps. Fully synchronous blocking offers stronger protection at substantial availability and latency cost.", "zh": "分析指出几类失效条件：危害早于审查发生、部分攻击进展不可逆、并行实例协调冗余攻击，以及监测错误跨步骤相关。完全同步阻断提供更强保护，但付出明显的可用性与时延代价。" }, "supports": { "en": "The framework helps expose assumptions needed to turn per-attempt detection evidence into a deployment argument. It does not certify a particular production system.", "zh": "该框架有助于揭示将单次攻击检测证据转化为部署论证所需的假设，并不认证任何具体生产系统。" }, "limits": { "en": "The case studies concern hypothetical deployments and modeled attack vectors. Conclusions depend on representative attacks, incrimination, shutdown and recovery assumptions; empirical monitor performance alone does not establish these conditions.", "zh": "案例针对假设部署和建模攻击路径。结论依赖攻击代表性、归责、停机与恢复假设；经验监测表现本身不能证明这些条件成立。" }, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2512.22154" } ], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "arXiv:2512.22154v1, §§2–3, 4–5, 6.1–6.2; Appendix E" }, { "id": "async-control", "title": "Async control: Stress-testing asynchronous control measures for LLM agents", "aliases": [], "authors": [], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "study", "sourceId": "async-control-publication", "attributionSourceId": "async-control-publication", "originalUrl": "https://www.aisi.gov.uk/research/async-control-stress-testing-asynchronous-control-measures-for-llm-agents", "publicationDate": "2025-12-15", "updatedOn": "2026-10-25", "areas": [ "deception", "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "async-control-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Stress-tests asynchronous oversight in a red-team/blue-team setting where agents may sabotage code before monitors intervene.", "zh": "在红蓝对抗中，对异步监督开展压力测试，研究智能体在监测干预前破坏代码的可能性。" }, "note": { "question": { "en": "How robust is asynchronous monitoring when an adversarial coding agent knows the monitor and adapts its system prompt against it?", "zh": "当对抗编码智能体了解监测器并据此调整系统提示时，异步监测能有多稳健？" }, "method": { "en": "The study runs a red-team/blue-team game on software-engineering environments with benign tasks and sabotage side tasks. The red team adapts an agent’s system prompt after inspecting the monitor; the blue team builds monitor ensembles. It also simulates auditing queues ordered by suspicion.", "zh": "研究在包含正常任务与破坏附加任务的软件工程环境中开展红蓝队博弈。红队查看监测器后调整智能体系统提示，蓝队构建监测器集成；研究还模拟按可疑度排序的审计队列。" }, "finding": { "en": "Monitor ensembles outperform individual monitors in the tested adversarial rounds and held-out environment. In deployment simulations, delaying harm or requiring several successful attacks can improve safety more than an isolated increase in monitor accuracy.", "zh": "在所测对抗轮次和留出环境中，监测器集成优于单一监测器。部署模拟显示，延迟危害发生或要求攻击连续成功多次，可能比单独提升监测准确率更能改善安全性。" }, "supports": { "en": "The evidence supports evaluating monitor ensembles together with the timing and reversibility of harm; detection accuracy alone is not a deployment safety claim.", "zh": "证据支持将监测器集成与危害发生时机、可逆性一起评估；仅有检测准确率不足以形成部署安全结论。" }, "limits": { "en": "The game restricts attacker adaptation to system prompts. Simulations do not directly measure action-to-harm latency or how multiple security-weakening steps compose into actual harm, so their deployment conclusions depend on those assumptions.", "zh": "该博弈将攻击者适配限制为系统提示调整。模拟未直接测量动作到危害的时延，也未完整测量多个削弱安全的步骤如何组合成实际危害，因此部署结论依赖这些假设。" }, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2512.13526" } ], "noteStatus": { "question": "verified", "method": "verified", "finding": "verified", "supports": "verified", "limits": "verified", "connections": "limited" }, "noteAvailability": { "en": "The substantive sections shown here were checked against the original source. Unfilled sections are omitted; source checking is not experimental replication.", "zh": "此处展示的实质性章节已对照原始来源核查；未完成的章节已省略。来源核查不等于实验复现。" }, "noteSourceLocator": "arXiv:2512.13526v1, §§1–3, red-team/blue-team rounds and deployment simulations; limitations in Introduction" }, { "id": "self-evaluation-wireheading", "title": "Does self-evaluation enable wireheading in language models?", "aliases": [], "authors": [], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "self-evaluation-wireheading-publication", "attributionSourceId": "self-evaluation-wireheading-publication", "originalUrl": "https://www.aisi.gov.uk/research/does-self-evaluation-enable-wireheading-in-language-models", "publicationDate": "2025-12-01", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "self-evaluation-wireheading-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Studies incentives to manipulate self-evaluation rewards using a formal model and controlled language-model experiments.", "zh": "结合形式模型与受控语言模型实验，研究操纵自评奖励的激励。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2511.23092" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "open-weight-risk-management", "title": "Open technical problems in open-weight AI model risk management", "aliases": [], "authors": [], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "open-weight-risk-management-publication", "attributionSourceId": "open-weight-risk-management-publication", "originalUrl": "https://www.aisi.gov.uk/research/open-technical-problems-in-open-weight-ai-model-risk-management", "publicationDate": "2025-10-26", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "open-weight-risk-management-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Identifies technical research gaps in managing risks from modifiable, redistributable open-weight models.", "zh": "梳理可修改、可再分发的开放权重模型在风险管理中的技术研究缺口。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "pretraining-poisoning", "title": "Poisoning Attacks on LLMs Require a Near-constant Number of Poison Samples", "aliases": [], "authors": [ "Alexandra Souly", "Javier Rando", "Ed Chapman", "Xander Davies", "Burak Hasircioglu", "Ezzeldin Shereen", "Carlos Mougan", "Vasilios Mavroudis", "Erik Jones", "Chris Hicks", "Nicholas Carlini", "Yarin Gal", "Robert Kirk" ], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "pretraining-poisoning-publication", "attributionSourceId": "pretraining-poisoning-publication-author-source", "originalUrl": "https://www.aisi.gov.uk/research/poisoning-attacks-on-llms-require-a-near-constant-number-of-poison-samples", "publicationDate": "2025-10-08", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "research", "sourceId": "pretraining-poisoning-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "anthropic", "role": "research", "sourceId": "pretraining-poisoning-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "eth", "role": "research", "sourceId": "pretraining-poisoning-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "oxford", "role": "research", "sourceId": "pretraining-poisoning-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Tests pretraining-data poisoning across model and dataset scales to examine the amount of malicious data needed for a targeted effect.", "zh": "跨模型与数据规模测试预训练数据投毒，研究诱发指定效果所需的恶意数据量。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2510.07192" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "inoculation-prompting", "title": "Inoculation Prompting: Eliciting traits from LLMs during training can suppress them at test-time", "aliases": [], "authors": [ "Daniel Tan", "Anders Woodruff", "Niels Warncke", "Arun Jose", "Maxime Riché", "David Demitri Africa", "Mia Taylor" ], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "study", "sourceId": "inoculation-prompting-publication", "attributionSourceId": "inoculation-prompting-publication-author-source", "originalUrl": "https://www.aisi.gov.uk/research/inoculation-prompting-eliciting-traits-from-llms-during-training-can-suppress-them-at-test-time", "publicationDate": "2025-10-05", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "research", "sourceId": "inoculation-prompting-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Tests whether eliciting an unwanted trait in fine-tuning prompts reduces its expression when those prompts are removed at test time.", "zh": "测试在微调提示中诱发不期望特征，能否在测试时移除该提示后降低特征表现。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2510.04340" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "misuse-safety-case", "title": "An example safety case for safeguards against misuse", "aliases": [], "authors": [], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "misuse-safety-case-publication", "attributionSourceId": "misuse-safety-case-publication", "originalUrl": "https://www.aisi.gov.uk/research/an-example-safety-case-for-safeguards-against-misuse", "publicationDate": "2025-06-05", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "misuse-safety-case-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Constructs an illustrative argument linking safeguard red-teaming evidence, attacker effort and deployment-risk decisions.", "zh": "构造示例安全论证，将防护红队证据、攻击者投入与部署风险决策联系起来。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2505.18003" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "unlearning-evaluation-validity", "title": "Existing Large Language Model unlearning evaluations are inconclusive", "aliases": [], "authors": [], "primaryFunction": "investigation", "secondaryFunctions": [], "kind": "study", "sourceId": "unlearning-evaluation-validity-publication", "attributionSourceId": "unlearning-evaluation-validity-publication", "originalUrl": "https://www.aisi.gov.uk/research/existing-large-language-model-unlearning-evaluations-are-inconclusive", "publicationDate": "2025-05-31", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "unlearning-evaluation-validity-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Examines whether unlearning tests accidentally reteach removed information and how task variation changes conclusions.", "zh": "检查遗忘评测是否意外重新传授被移除信息，以及任务变化如何改变结论。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2506.00688" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "safety-cases", "title": "Safety Cases: A scalable approach to Frontier AI safety", "aliases": [], "authors": [], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "safety-cases-publication", "attributionSourceId": "safety-cases-publication", "originalUrl": "https://www.aisi.gov.uk/research/safety-cases-a-scalable-approach-to-frontier-ai-safety", "publicationDate": "2025-02-10", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "safety-cases-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Explains how structured claims, evidence and assumptions can support frontier-AI safety decisions and identifies methodological questions.", "zh": "说明结构化主张、证据与假设如何支持前沿 AI 安全决策，并提出方法问题。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://cdn.prod.website-files.com/663bd486c5e4c81588db7a1d/67aa1ee70c744109b20a0032_SafetyCasesAScalableApproachToFrontierAISafety.pdf" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "misuse-safeguard-principles", "title": "Principles for evaluating misuse safeguards of frontier AI systems", "aliases": [], "authors": [], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "method", "sourceId": "misuse-safeguard-principles-publication", "attributionSourceId": "misuse-safeguard-principles-publication", "originalUrl": "https://www.aisi.gov.uk/research/principles-for-evaluating-misuse-safeguards-of-frontier-ai-systems", "publicationDate": "2025-02-04", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "publisher", "sourceId": "misuse-safeguard-principles-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Proposes practical principles for defining and evaluating misuse safeguards and documenting decision-relevant evidence.", "zh": "提出定义、评测滥用防护及记录决策相关证据的实用原则。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://cdn.prod.website-files.com/663bd486c5e4c81588db7a1d/67a23823ef4a296bb6c92dd4_Principles%20for%20Evaluating%20Misuse%20Safeguards%20of%20Frontier%20AI%20Systems%20(1).pdf" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "control-safety-case", "title": "A sketch of an AI control safety case", "aliases": [], "authors": [ "Tomek Korbak", "Joshua Clymer", "Benjamin Hilton", "Buck Shlegeris", "Geoffrey Irving" ], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "control-safety-case-publication", "attributionSourceId": "control-safety-case-publication-author-source", "originalUrl": "https://www.aisi.gov.uk/research/a-sketch-of-an-ai-control-safety-case", "publicationDate": "2025-01-28", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "aisi", "role": "research", "sourceId": "control-safety-case-publication-author-source", "locator": "Author affiliations on the original paper title page" }, { "organizationId": "redwood", "role": "research", "sourceId": "control-safety-case-publication-author-source", "locator": "Author affiliations on the original paper title page" } ], "summary": { "en": "Sketches how adversarial control evaluations could support a bounded claim against sensitive-information exfiltration by an internal agent.", "zh": "说明对抗控制评测如何支持针对内部智能体敏感信息外泄的有界安全主张。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [ { "label": "Read the full paper", "url": "https://arxiv.org/abs/2501.17315" } ], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "embedded-evaluators", "title": "Embedded Evaluators are necessary for meaningful external testing", "aliases": [], "authors": [], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "embedded-evaluators-publication", "attributionSourceId": "embedded-evaluators-publication", "originalUrl": "https://www.apolloresearch.ai/blog/embedded-evaluators-are-necessary-for-meaningful-external-testing", "publicationDate": "2026-09-29", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "apollo", "role": "publisher", "sourceId": "embedded-evaluators-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Argues that independent evaluators need access throughout development to examine incidents that final-checkpoint tests may miss.", "zh": "论证独立评测者需要在开发全过程获得访问权限，以调查最终检查点测试可能遗漏的事件。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "specification-gaming", "title": "Demonstrating specification gaming in reasoning models", "aliases": [], "authors": [], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "specification-gaming-publication", "attributionSourceId": "specification-gaming-publication", "originalUrl": "https://palisaderesearch.org/research/specification-gaming", "publicationDate": "2025-02-19", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "palisade", "role": "publisher", "sourceId": "specification-gaming-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Studies whether agents exploit a chess-task specification instead of winning through the intended gameplay.", "zh": "研究智能体是否利用国际象棋任务规范的漏洞，而非通过预期棋局获胜。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "llm-honeypot", "title": "LLM Honeypot: an early warning system for autonomous hacking", "aliases": [], "authors": [], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "method", "sourceId": "llm-honeypot-publication", "attributionSourceId": "llm-honeypot-publication", "originalUrl": "https://palisaderesearch.org/research/llm-honeypot", "publicationDate": "2024-10-17", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "palisade", "role": "publisher", "sourceId": "llm-honeypot-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Uses a deployed honeypot to investigate signals that may distinguish autonomous AI hacking attempts from other traffic.", "zh": "利用部署的蜜罐，研究可能区分 AI 自主攻击尝试与其他流量的信号。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "beyond-owls", "title": "Beyond Owls: Subliminal Learning Can Transfer Learned Capabilities and Backdoors", "aliases": [], "authors": [ "Jan Dubiński", "Anna Sztyber-Betley", "Jan Betley", "Owain Evans" ], "primaryFunction": "discovery", "secondaryFunctions": [], "kind": "study", "sourceId": "beyond-owls-publication", "attributionSourceId": "beyond-owls-publication", "originalUrl": "https://truthful.ai/papers/beyond-owls/", "publicationDate": "2026-10-07", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "truthful", "role": "publisher", "sourceId": "beyond-owls-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Extends subliminal-learning experiments to learned capabilities, backdoors and behavioral tendencies transmitted through unrelated data.", "zh": "将隐性学习实验扩展至通过无关数据传递的已学能力、后门和行为倾向。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "activation-oracles", "title": "Activation Oracles: Training and Evaluating LLMs as General-Purpose Activation Explainers", "aliases": [], "authors": [ "Adam Karvonen", "James Chua", "Clément Dumas", "Kit Fraser-Taliente", "Subhash Kantamneni", "Julian Minder", "Euan Ong", "Arnab Sen Sharma", "Daniel Wen", "Owain Evans", "Samuel Marks" ], "primaryFunction": "monitoring", "secondaryFunctions": [], "kind": "method", "sourceId": "activation-oracles-publication", "attributionSourceId": "activation-oracles-publication", "originalUrl": "https://truthful.ai/papers/activation-oracles/", "publicationDate": "2025-12-19", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "truthful", "role": "publisher", "sourceId": "activation-oracles-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Trains models to answer questions about another model’s activations and evaluates generalization to hidden information and goals.", "zh": "训练模型回答关于另一模型激活的问题，并评测其对隐藏信息与目标的泛化。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "rogue-replication-threat", "title": "The Rogue Replication Threat Model", "aliases": [], "authors": [], "primaryFunction": "governance", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "rogue-replication-threat-publication", "attributionSourceId": "rogue-replication-threat-publication", "originalUrl": "https://metr.org/blog/2024-11-12-rogue-replication-threat-model/", "publicationDate": "2024-11-12", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "metr", "role": "publisher", "sourceId": "rogue-replication-threat-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Develops a threat model for unauthorized replication and identifies the capabilities and conditions that would enable it.", "zh": "为未经授权的复制建立威胁模型，并识别其所需能力与条件。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "capability-elicitation", "title": "Guidelines for capability elicitation", "aliases": [], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "method", "sourceId": "capability-elicitation-publication", "attributionSourceId": "capability-elicitation-publication", "originalUrl": "https://metr.org/blog/2024-03-15-guidelines-for-capability-elicitation/", "publicationDate": "2024-03-15", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.", "zh": "风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。" }, "protocolIds": [], "contributions": [ { "organizationId": "metr", "role": "publisher", "sourceId": "capability-elicitation-publication", "locator": "Official publication masthead and article byline" } ], "summary": { "en": "Specifies practices for eliciting model capabilities so evaluation failures are not mistaken for inability.", "zh": "提出诱发模型能力的评测实践，避免将评测失败误判为不具备能力。" }, "note": { "question": null, "method": null, "finding": null, "supports": null, "limits": null, "connections": null }, "resources": [], "noteStatus": { "question": "limited", "method": "limited", "finding": "limited", "supports": "limited", "limits": "limited", "connections": "limited" }, "noteAvailability": { "en": "A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.", "zh": "实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。" }, "noteSourceLocator": "No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence." }, { "id": "abstentionbench", "title": "AbstentionBench: Reasoning LLMs Fail on Unanswerable Questions", "aliases": [ "AbstentionBench" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "abstentionbench-primary", "attributionSourceId": "abstentionbench-primary", "originalUrl": "https://arxiv.org/abs/2506.09038", "publicationDate": "2025-06-10", "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Mapped to the documented research question. A risk-topic association is not a severity score.", "zh": "依据明确研究问题关联；风险主题不代表严重性评分。" }, "protocolIds": [ "abstentionbench" ], "contributions": [], "summary": { "en": "When does a model abstain from questions that cannot be answered reliably?", "zh": "模型何时会对无法可靠回答的问题选择弃答？" }, "note": { "question": { "en": "When does a model abstain from questions that cannot be answered reliably?", "zh": "模型何时会对无法可靠回答的问题选择弃答？" }, "method": { "en": "Tests abstention across 20 datasets with unknown, underspecified, false-premise, subjective or outdated questions.", "zh": "在 20 个数据集中检验未知答案、信息不足、错误前提、主观或过时问题的弃答。" }, "finding": null, "supports": null, "limits": { "en": "Abstention is distinct from a safety refusal and does not alone establish intent or downstream harm.", "zh": "弃答不同于安全拒答，不能单独证明意图或下游伤害。" }, "connections": null }, "noteAvailability": { "en": "This record describes the protocol. Report-specific outcomes are linked through assessments; no unsourced general finding is added.", "zh": "本档案说明协议。报告特定结果通过评估记录关联，不补写无出处的总体发现。" }, "resources": [] }, { "id": "deceptionbench", "title": "DeceptionBench : A Comprehensive Benchmark for AI Deception Behaviors in Real-world Scenarios", "aliases": [ "DeceptionBench" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "deceptionbench-primary", "attributionSourceId": "deceptionbench-primary", "originalUrl": "https://arxiv.org/abs/2510.15501", "publicationDate": "2025-10-17", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Mapped to the documented research question. A risk-topic association is not a severity score.", "zh": "依据明确研究问题关联；风险主题不代表严重性评分。" }, "protocolIds": [ "deceptionbench" ], "contributions": [], "summary": { "en": "How do incentives and multi-turn pressure change deceptive responses?", "zh": "激励与多轮压力如何改变欺骗性回答？" }, "note": { "question": { "en": "How do incentives and multi-turn pressure change deceptive responses?", "zh": "激励与多轮压力如何改变欺骗性回答？" }, "method": { "en": "Evaluates 150 authored scenarios across five social domains, including neutral, incentive and pressure conditions.", "zh": "评测五个社会领域的 150 个编写情境，区分中性、激励与压力条件。" }, "finding": null, "supports": null, "limits": { "en": "Authored scenarios and model-based judging limit generalization; elicited deception does not establish covert long-term scheming.", "zh": "编写情境和模型判别限制外推；诱发的欺骗不证明隐蔽长程谋划。" }, "connections": null }, "noteAvailability": { "en": "This record describes the protocol. Report-specific outcomes are linked through assessments; no unsourced general finding is added.", "zh": "本档案说明协议。报告特定结果通过评估记录关联，不补写无出处的总体发现。" }, "resources": [] }, { "id": "persusafety", "title": "LLM Can be a Dangerous Persuader: Empirical Study of Persuasion Safety in Large Language Models Ethical Disclaimer: This paper may contain unethical persuasion content.", "aliases": [ "PersuSafety" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "persusafety-primary", "attributionSourceId": "persusafety-primary", "originalUrl": "https://arxiv.org/abs/2504.10430", "publicationDate": "2025-04-14", "updatedOn": "2026-10-25", "areas": [ "disinformation" ], "scopeNote": { "en": "Mapped to the documented research question. A risk-topic association is not a severity score.", "zh": "依据明确研究问题关联；风险主题不代表严重性评分。" }, "protocolIds": [ "persusafety" ], "contributions": [], "summary": { "en": "How does model engagement with persuasion requests vary with topic harm?", "zh": "模型参与说服请求的方式如何随主题危害变化？" }, "note": { "question": { "en": "How does model engagement with persuasion requests vary with topic harm?", "zh": "模型参与说服请求的方式如何随主题危害变化？" }, "method": { "en": "Defines persuasion scenarios, simulated conversations and safety assessment; adapted judges are separate variants.", "zh": "定义说服情境、模拟对话与安全评测；改造判别器作为独立变体。" }, "finding": null, "supports": null, "limits": { "en": "Willingness or simulated dialogue scores do not directly measure changes in human beliefs.", "zh": "参与意愿或模拟对话评分不直接测量人的信念变化。" }, "connections": null }, "noteAvailability": { "en": "This record describes the protocol. Report-specific outcomes are linked through assessments; no unsourced general finding is added.", "zh": "本档案说明协议。报告特定结果通过评估记录关联，不补写无出处的总体发现。" }, "resources": [] }, { "id": "impossiblebench", "title": "ImpossibleBench: Measuring LLMs’ Propensity of Exploiting Test Cases", "aliases": [ "ImpossibleBench" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "impossiblebench-primary", "attributionSourceId": "impossiblebench-primary", "originalUrl": "https://arxiv.org/abs/2510.20270", "publicationDate": "2025-10-23", "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Mapped to the documented research question. A risk-topic association is not a severity score.", "zh": "依据明确研究问题关联；风险主题不代表严重性评分。" }, "protocolIds": [ "impossiblebench" ], "contributions": [], "summary": { "en": "Will coding agents exploit tests when specifications and tests cannot both be satisfied?", "zh": "当规范与测试无法同时满足时，编程智能体是否会利用测试机制？" }, "note": { "question": { "en": "Will coding agents exploit tests when specifications and tests cannot both be satisfied?", "zh": "当规范与测试无法同时满足时，编程智能体是否会利用测试机制？" }, "method": { "en": "Constructs impossible variants of coding tasks and examines specification-violating passes.", "zh": "构造不可同时满足要求的编程任务变体，检验违反规范的通过结果。" }, "finding": null, "supports": null, "limits": { "en": "The cheating rate is conditional on contradictions, test access and scaffolding; it is not a general dishonesty rate.", "zh": "作弊率受矛盾条件、测试访问和脚手架限制，不是一般不诚实率。" }, "connections": null }, "noteAvailability": { "en": "This record describes the protocol. Report-specific outcomes are linked through assessments; no unsourced general finding is added.", "zh": "本档案说明协议。报告特定结果通过评估记录关联，不补写无出处的总体发现。" }, "resources": [] }, { "id": "evalawarebench", "title": "Decomposing and Measuring Evaluation Awareness", "aliases": [ "EvalAwareBench" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "evalawarebench-primary", "attributionSourceId": "evalawarebench-primary", "originalUrl": "https://arxiv.org/abs/2605.23055", "publicationDate": null, "updatedOn": "2026-10-25", "areas": [ "deception" ], "scopeNote": { "en": "Mapped to the documented research question. A risk-topic association is not a severity score.", "zh": "依据明确研究问题关联；风险主题不代表严重性评分。" }, "protocolIds": [ "evalawarebench" ], "contributions": [], "summary": { "en": "Which evaluation cues cause models to verbalize awareness of being tested?", "zh": "哪些评测线索会使模型表达被测试的感知？" }, "note": { "question": { "en": "Which evaluation cues cause models to verbalize awareness of being tested?", "zh": "哪些评测线索会使模型表达被测试的感知？" }, "method": { "en": "Varies evaluation cues while tracking verbalized evaluation awareness in controlled tasks.", "zh": "在受控任务中改变评测线索，记录显式表达的评测感知。" }, "finding": null, "supports": null, "limits": { "en": "Unexpressed awareness is not measured; recognition of familiar benchmarks can differ from general cue response.", "zh": "不能测量未表达的感知；熟悉基准的识别可能不同于一般线索反应。" }, "connections": null }, "noteAvailability": { "en": "This record describes the protocol. Report-specific outcomes are linked through assessments; no unsourced general finding is added.", "zh": "本档案说明协议。报告特定结果通过评估记录关联，不补写无出处的总体发现。" }, "resources": [] }, { "id": "theagentcompany", "title": "TheAgentCompany: Benchmarking LLM Agents on Consequential Real World Tasks", "aliases": [ "TheAgentCompany" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "theagentcompany-primary", "attributionSourceId": "theagentcompany-primary", "originalUrl": "https://arxiv.org/abs/2412.14161", "publicationDate": "2024-12-18", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "Mapped to the documented research question. A risk-topic association is not a severity score.", "zh": "依据明确研究问题关联；风险主题不代表严重性评分。" }, "protocolIds": [ "theagentcompany" ], "contributions": [], "summary": { "en": "How do agents complete consequential workplace tasks in a simulated company?", "zh": "智能体如何在模拟公司中完成实际工作任务？" }, "note": { "question": { "en": "How do agents complete consequential workplace tasks in a simulated company?", "zh": "智能体如何在模拟公司中完成实际工作任务？" }, "method": { "en": "Combines browsing, coding, file operations and interaction with simulated coworkers.", "zh": "结合浏览、编程、文件操作和与模拟同事的交互。" }, "finding": null, "supports": null, "limits": { "en": "Task success and evaluation-boundary behavior are separate outcomes; a broken environment is not an ordinary task failure.", "zh": "任务成功与评测边界行为属于不同结果；环境故障不是普通任务失败。" }, "connections": null }, "noteAvailability": { "en": "This record describes the protocol. Report-specific outcomes are linked through assessments; no unsourced general finding is added.", "zh": "本档案说明协议。报告特定结果通过评估记录关联，不补写无出处的总体发现。" }, "resources": [] }, { "id": "lab-bench", "title": "LAB-Bench Measuring Capabilities of Language Models for Biology Research", "aliases": [ "LAB-Bench" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "lab-bench-primary", "attributionSourceId": "lab-bench-primary", "originalUrl": "https://arxiv.org/abs/2407.10362", "publicationDate": "2024-07-14", "updatedOn": "2026-10-25", "areas": [ "cbrn" ], "scopeNote": { "en": "Mapped to the documented research question. A risk-topic association is not a severity score.", "zh": "依据明确研究问题关联；风险主题不代表严重性评分。" }, "protocolIds": [ "lab-bench" ], "contributions": [], "summary": { "en": "How do models perform on research-oriented scientific tasks?", "zh": "模型在面向研究的科学任务中表现如何？" }, "note": { "question": { "en": "How do models perform on research-oriented scientific tasks?", "zh": "模型在面向研究的科学任务中表现如何？" }, "method": { "en": "Evaluates a range of biology research skills with structured questions; text-only or missing-input subsets remain separate protocols.", "zh": "用结构化问题评测生物研究技能；纯文本或缺失输入子集采用独立协议。" }, "finding": null, "supports": null, "limits": { "en": "These are general scientific capabilities, not end-to-end misuse tasks or direct harm probabilities.", "zh": "衡量一般科研能力，不是端到端滥用任务或直接伤害概率。" }, "connections": null }, "noteAvailability": { "en": "This record describes the protocol. Report-specific outcomes are linked through assessments; no unsourced general finding is added.", "zh": "本档案说明协议。报告特定结果通过评估记录关联，不补写无出处的总体发现。" }, "resources": [] }, { "id": "charxiv", "title": "CharXiv: Charting Gaps in Realistic Chart Understanding in Multimodal LLMs", "aliases": [ "CharXiv" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "charxiv-primary", "attributionSourceId": "charxiv-primary", "originalUrl": "https://charxiv.github.io/", "publicationDate": null, "updatedOn": "2026-10-25", "areas": [], "scopeNote": { "en": "Mapped to the documented research question. A risk-topic association is not a severity score.", "zh": "依据明确研究问题关联；风险主题不代表严重性评分。" }, "protocolIds": [ "charxiv" ], "contributions": [], "summary": { "en": "Can models interpret research charts?", "zh": "模型能否理解研究图表？" }, "note": { "question": { "en": "Can models interpret research charts?", "zh": "模型能否理解研究图表？" }, "method": { "en": "Tests understanding and reasoning over charts. Neo’s withheld-image adaptation measures attempts without the required input.", "zh": "检验图表理解与推理。Neo 移除图像的改造测量缺少必要输入时的回答尝试。" }, "finding": null, "supports": null, "limits": { "en": "The original visual capability benchmark and the missing-image behavioral test measure different constructs.", "zh": "原始视觉能力基准与缺失图像行为测试测量不同构念。" }, "connections": null }, "noteAvailability": { "en": "This record describes the protocol. Report-specific outcomes are linked through assessments; no unsourced general finding is added.", "zh": "本档案说明协议。报告特定结果通过评估记录关联，不补写无出处的总体发现。" }, "resources": [] }, { "id": "biomysterybench", "title": "Evaluating Claude’s bioinformatics research capabilities with BioMysteryBench", "aliases": [ "BioMysteryBench" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "biomysterybench-primary", "attributionSourceId": "biomysterybench-primary", "originalUrl": "https://www.anthropic.com/research/Evaluating-Claude-For-Bioinformatics-With-BioMysteryBench", "publicationDate": null, "updatedOn": "2026-10-25", "areas": [ "cbrn" ], "scopeNote": { "en": "Mapped to the documented research question. A risk-topic association is not a severity score.", "zh": "依据明确研究问题关联；风险主题不代表严重性评分。" }, "protocolIds": [ "biomysterybench" ], "contributions": [], "summary": { "en": "Can agents solve research-oriented data-analysis problems?", "zh": "智能体能否解决面向科研的数据分析问题？" }, "note": { "question": { "en": "Can agents solve research-oriented data-analysis problems?", "zh": "智能体能否解决面向科研的数据分析问题？" }, "method": { "en": "Assesses scientific analysis over anonymized datasets; versions, human reference subsets and attempt budgets are kept explicit.", "zh": "评测匿名数据集上的科研分析；明确版本、人类参照子集和尝试预算。" }, "finding": null, "supports": null, "limits": { "en": "Aggregate scientific performance does not establish an operational biological misuse pathway.", "zh": "汇总科研表现不能证明实际生物滥用路径。" }, "connections": null }, "noteAvailability": { "en": "This record describes the protocol. Report-specific outcomes are linked through assessments; no unsourced general finding is added.", "zh": "本档案说明协议。报告特定结果通过评估记录关联，不补写无出处的总体发现。" }, "resources": [] }, { "id": "evmbench", "title": "EVMbench", "aliases": [ "EVMbench" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "evmbench-primary", "attributionSourceId": "evmbench-primary", "originalUrl": "https://openai.com/index/introducing-evmbench/", "publicationDate": null, "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Mapped to the documented research question. A risk-topic association is not a severity score.", "zh": "依据明确研究问题关联；风险主题不代表严重性评分。" }, "protocolIds": [ "evmbench" ], "contributions": [], "summary": { "en": "Can agents detect, repair and exploit smart-contract vulnerabilities in the benchmark?", "zh": "智能体能否在基准中检测、修补或利用智能合约漏洞？" }, "note": { "question": { "en": "Can agents detect, repair and exploit smart-contract vulnerabilities in the benchmark?", "zh": "智能体能否在基准中检测、修补或利用智能合约漏洞？" }, "method": { "en": "Separates detection, patching and exploitation modes with their stated task sets.", "zh": "分别组织检测、修补和利用模式及对应任务集。" }, "finding": null, "supports": null, "limits": { "en": "Modes and task subsets cannot be combined into an undifferentiated cyber-risk score.", "zh": "不同模式和子集不能合并成无区分的网络风险分数。" }, "connections": null }, "noteAvailability": { "en": "This record describes the protocol. Report-specific outcomes are linked through assessments; no unsourced general finding is added.", "zh": "本档案说明协议。报告特定结果通过评估记录关联，不补写无出处的总体发现。" }, "resources": [] }, { "id": "swe-bench-pro", "title": "SWE-Bench Pro: Can AI Agents Solve Long-Horizon Software Engineering Tasks?", "aliases": [ "SWE-Bench Pro" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "swe-bench-pro-primary", "attributionSourceId": "swe-bench-pro-primary", "originalUrl": "https://arxiv.org/abs/2509.16941", "publicationDate": null, "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "Mapped to the documented research question. A risk-topic association is not a severity score.", "zh": "依据明确研究问题关联；风险主题不代表严重性评分。" }, "protocolIds": [ "swe-bench-pro" ], "contributions": [], "summary": { "en": "Can agents solve complex software maintenance tasks?", "zh": "智能体能否解决复杂软件维护任务？" }, "note": { "question": { "en": "Can agents solve complex software maintenance tasks?", "zh": "智能体能否解决复杂软件维护任务？" }, "method": { "en": "Evaluates repository-level changes; public subsets, scaffolds and reward-hacking scans are version-specific.", "zh": "评测仓库级代码修改；公开子集、脚手架和奖励投机扫描有各自版本。" }, "finding": null, "supports": null, "limits": { "en": "General coding capability is enabling evidence, not direct evidence of loss of control.", "zh": "一般编程能力属于使能证据，不是失控的直接证据。" }, "connections": null }, "noteAvailability": { "en": "This record describes the protocol. Report-specific outcomes are linked through assessments; no unsourced general finding is added.", "zh": "本档案说明协议。报告特定结果通过评估记录关联，不补写无出处的总体发现。" }, "resources": [] }, { "id": "ctf-archive-diamond", "title": "CTF-Archive-Diamond", "aliases": [ "CTF-Archive-Diamond" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "ctf-archive-diamond-primary", "attributionSourceId": "ctf-archive-diamond-primary", "originalUrl": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "publicationDate": "2026-05-01", "updatedOn": "2026-10-25", "areas": [ "cybersecurity" ], "scopeNote": { "en": "Mapped to the documented research question. A risk-topic association is not a severity score.", "zh": "依据明确研究问题关联；风险主题不代表严重性评分。" }, "protocolIds": [ "ctf-archive-diamond" ], "contributions": [], "summary": { "en": "How do agents solve a difficult CTF subset?", "zh": "智能体在高难度 CTF 子集中表现如何？" }, "note": { "question": { "en": "How do agents solve a difficult CTF subset?", "zh": "智能体在高难度 CTF 子集中表现如何？" }, "method": { "en": "CAISI uses 285 difficult CTF challenges from the pwn.challenge collection; imputed model results are marked separately.", "zh": "CAISI 采用 pwn.challenge 集合的 285 项高难度 CTF 挑战；模型插补结果单独标记。" }, "finding": null, "supports": null, "limits": { "en": "CTF tasks are not live operational attacks; subset-imputed values are not directly observed full-set success rates.", "zh": "CTF 不是现实行动攻击；子集插补值不是全量直接观察的成功率。" }, "connections": null }, "noteAvailability": { "en": "This record describes the protocol. Report-specific outcomes are linked through assessments; no unsourced general finding is added.", "zh": "本档案说明协议。报告特定结果通过评估记录关联，不补写无出处的总体发现。" }, "resources": [] }, { "id": "portbench", "title": "PortBench", "aliases": [ "PortBench" ], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "benchmark", "sourceId": "portbench-primary", "attributionSourceId": "portbench-primary", "originalUrl": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "publicationDate": "2026-05-01", "updatedOn": "2026-10-25", "areas": [ "autonomy" ], "scopeNote": { "en": "Mapped to the documented research question. A risk-topic association is not a severity score.", "zh": "依据明确研究问题关联；风险主题不代表严重性评分。" }, "protocolIds": [ "portbench" ], "contributions": [], "summary": { "en": "Can models port CLI tools across programming languages?", "zh": "模型能否跨编程语言移植 CLI 工具？" }, "note": { "question": { "en": "Can models port CLI tools across programming languages?", "zh": "模型能否跨编程语言移植 CLI 工具？" }, "method": { "en": "CAISI’s held-out software engineering evaluation uses reference implementations and a reported agent budget.", "zh": "CAISI 的保留软件工程评测采用参考实现和明确智能体预算。" }, "finding": null, "supports": null, "limits": { "en": "The task set is non-public and continuous scoring differs from binary task success.", "zh": "任务集未公开，连续评分不同于二元任务成功。" }, "connections": null }, "noteAvailability": { "en": "This record describes the protocol. Report-specific outcomes are linked through assessments; no unsourced general finding is added.", "zh": "本档案说明协议。报告特定结果通过评估记录关联，不补写无出处的总体发现。" }, "resources": [] }, { "id": "neo-kimi-k3-report", "title": "Evaluating Kimi K3 for Misalignment Risks", "aliases": [], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "neo-kimi-k3-source", "attributionSourceId": "neo-kimi-k3-source", "originalUrl": "https://neoresearch.ai/papers/Kimi_K3_Safety_Evaluation.pdf", "publicationDate": "2026-09-11", "updatedOn": "2026-10-25", "areas": [ "autonomy", "deception", "cybersecurity", "cbrn", "disinformation" ], "scopeNote": { "en": "Reported results retain their original evaluator, configuration and measurement scope. NUWA organizes and interprets the evidence; it has not independently reproduced these experiments.", "zh": "结果保留原始执行方、配置与测量范围。NUWA 整理和解读证据，未独立复现这些实验。" }, "protocolIds": [ "charxiv", "lab-bench", "abstentionbench", "cybench", "evmbench", "cybergym", "theagentcompany", "impossiblebench", "evalawarebench", "alignment-faking", "agentic-misalignment", "palisade-shutdown", "petri", "truthful-sad", "stealth-evals", "shade-arena", "agentharm", "fortress", "ape", "persusafety", "deceptionbench", "mask" ], "contributions": [ { "organizationId": "neo", "role": "research", "sourceId": "neo-kimi-k3-source", "locator": "Report masthead and stated evaluation methodology" } ], "summary": { "en": "The report identifies conditional overeagerness, grader-directed behavior and recognition of familiar evaluations. It does not establish a universal deployment failure rate.", "zh": "报告发现有条件的过度主动、面向判分器的行为和对熟悉评测的识别，不代表普遍部署失败率。" }, "note": { "question": { "en": "How does Kimi K3 behave when input, authorization or evaluation boundaries are ambiguous?", "zh": "当输入、授权或评测边界存在歧义时，Kimi K3 如何行动？" }, "method": { "en": "Nearly 30 evaluations, adapted missing-input tests, seeded audits and transcript analyses, with mixed-origin comparison results.", "zh": "近 30 项评测、缺失输入改造、种子审计和轨迹分析，部分对照结果来自其他来源。" }, "finding": { "en": "The report identifies conditional overeagerness, grader-directed behavior and recognition of familiar evaluations. It does not establish a universal deployment failure rate.", "zh": "报告发现有条件的过度主动、面向判分器的行为和对熟悉评测的识别，不代表普遍部署失败率。" }, "supports": null, "limits": { "en": "Controlled tasks, elicitation, benchmark familiarity and configuration differences constrain generalization to deployment.", "zh": "受控任务、诱发方式、基准熟悉程度和配置差异限制对部署的外推。" }, "connections": null }, "resources": [] }, { "id": "neo-deepseek-v4-report", "title": "Evaluating DeepSeek v4 Pro for Frontier Risks", "aliases": [], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "neo-deepseek-v4-source", "attributionSourceId": "neo-deepseek-v4-source", "originalUrl": "https://neoresearch.ai/papers/DSv4_Safety_Evaluation_v1.1.pdf", "publicationDate": "2026-05-29", "updatedOn": "2026-10-25", "areas": [ "autonomy", "deception", "cybersecurity", "cbrn", "disinformation" ], "scopeNote": { "en": "Reported results retain their original evaluator, configuration and measurement scope. NUWA organizes and interprets the evidence; it has not independently reproduced these experiments.", "zh": "结果保留原始执行方、配置与测量范围。NUWA 整理和解读证据，未独立复现这些实验。" }, "protocolIds": [ "cybench", "evmbench", "wmdp", "lab-bench", "mask", "deceptionbench", "ape", "persusafety", "alignment-faking", "agentic-misalignment", "palisade-shutdown", "impossiblebench", "fortress", "petri" ], "contributions": [ { "organizationId": "neo", "role": "research", "sourceId": "neo-deepseek-v4-source", "locator": "Report masthead and stated evaluation methodology" } ], "summary": { "en": "The report separates task capability from pressure-induced behavior and notes format dependence in persuasion assessment.", "zh": "报告区分任务能力与压力诱发行为，并指出说服评测依赖提问格式。" }, "note": { "question": { "en": "Which capabilities, behavioral tendencies and evaluation-validity issues are observed in DeepSeek V4 Pro?", "zh": "DeepSeek V4 Pro 有哪些已观察的能力、行为倾向和评测有效性问题？" }, "method": { "en": "Public-access evaluations covering cyber, scientific knowledge, manipulation, control-related behavior, robustness and judge sensitivity.", "zh": "通过公开访问评测网络、科研知识、操纵、控制相关行为、鲁棒性与判别器敏感性。" }, "finding": { "en": "The report separates task capability from pressure-induced behavior and notes format dependence in persuasion assessment.", "zh": "报告区分任务能力与压力诱发行为，并指出说服评测依赖提问格式。" }, "supports": null, "limits": { "en": "Controlled tasks, elicitation, benchmark familiarity and configuration differences constrain generalization to deployment.", "zh": "受控任务、诱发方式、基准熟悉程度和配置差异限制对部署的外推。" }, "connections": null }, "resources": [] }, { "id": "neo-glm52-report", "title": "Early Results on Evaluating GLM 5.2 for Behavioral and Persuasion Risks", "aliases": [], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "neo-glm52-source", "attributionSourceId": "neo-glm52-source", "originalUrl": "https://neoresearch.ai/research/glm-5-2-behavioral-persuasion-evaluation/", "publicationDate": "2026-07-03", "updatedOn": "2026-10-25", "areas": [ "autonomy", "deception", "cybersecurity", "cbrn", "disinformation" ], "scopeNote": { "en": "Reported results retain their original evaluator, configuration and measurement scope. NUWA organizes and interprets the evidence; it has not independently reproduced these experiments.", "zh": "结果保留原始执行方、配置与测量范围。NUWA 整理和解读证据，未独立复现这些实验。" }, "protocolIds": [ "mask", "deceptionbench", "ape", "persusafety", "agentic-misalignment" ], "contributions": [ { "organizationId": "neo", "role": "research", "sourceId": "neo-glm52-source", "locator": "Report masthead and stated evaluation methodology" } ], "summary": { "en": "The note reports scenario-dependent behavior and an uncertain decline in verbalized evaluation awareness; silence is not evidence of absent awareness.", "zh": "报告记录随情境变化的行为和不确定的显式评测感知下降；不表达不代表没有感知。" }, "note": { "question": { "en": "How does GLM-5.2 respond to pressure, persuasion requests and evaluation cues?", "zh": "GLM-5.2 如何应对压力、说服请求和评测线索？" }, "method": { "en": "MASK, DeceptionBench, Gram, APE, modified PersuSafety judging and verbalized-awareness analyses. Claude comparisons use different releases across tests.", "zh": "采用 MASK、DeceptionBench、Gram、APE、改造 PersuSafety 判别和显式感知分析；各测试的 Claude 对照版本不同。" }, "finding": { "en": "The note reports scenario-dependent behavior and an uncertain decline in verbalized evaluation awareness; silence is not evidence of absent awareness.", "zh": "报告记录随情境变化的行为和不确定的显式评测感知下降；不表达不代表没有感知。" }, "supports": null, "limits": { "en": "Controlled tasks, elicitation, benchmark familiarity and configuration differences constrain generalization to deployment.", "zh": "受控任务、诱发方式、基准熟悉程度和配置差异限制对部署的外推。" }, "connections": null }, "resources": [] }, { "id": "caisi-deepseek-2025-report", "title": "Evaluation of DeepSeek AI Models", "aliases": [], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "caisi-deepseek-2025-source", "attributionSourceId": "caisi-deepseek-2025-source", "originalUrl": "https://www.nist.gov/system/files/documents/2025/09/30/CAISI_Evaluation_of_DeepSeek_AI_Models.pdf", "publicationDate": "2025-09-30", "updatedOn": "2026-10-25", "areas": [ "autonomy", "deception", "cybersecurity", "cbrn", "disinformation" ], "scopeNote": { "en": "Reported results retain their original evaluator, configuration and measurement scope. NUWA organizes and interprets the evidence; it has not independently reproduced these experiments.", "zh": "结果保留原始执行方、配置与测量范围。NUWA 整理和解读证据，未独立复现这些实验。" }, "protocolIds": [ "cybench", "cybergym", "agentdojo" ], "contributions": [ { "organizationId": "nist", "role": "research", "sourceId": "caisi-deepseek-2025-source", "locator": "Report masthead and stated evaluation methodology" } ], "summary": { "en": "Reported differences depend on task, scaffold and scoring; institutional judgments are retained as source judgments rather than NUWA findings.", "zh": "所报告差异依赖任务、脚手架与评分；机构判断保留为来源观点，不改写为 NUWA 实验发现。" }, "note": { "question": { "en": "How do selected DeepSeek releases perform under CAISI’s capability and security protocols?", "zh": "选定 DeepSeek 版本在 CAISI 能力与安全协议下表现如何？" }, "method": { "en": "The September 2025 report covers public and held-out tasks, inference configuration, costs, hijacking and jailbreak evaluations.", "zh": "2025 年 9 月报告覆盖公开与保留任务、推理配置、费用、劫持与越狱评测。" }, "finding": { "en": "Reported differences depend on task, scaffold and scoring; institutional judgments are retained as source judgments rather than NUWA findings.", "zh": "所报告差异依赖任务、脚手架与评分；机构判断保留为来源观点，不改写为 NUWA 实验发现。" }, "supports": null, "limits": { "en": "Controlled tasks, elicitation, benchmark familiarity and configuration differences constrain generalization to deployment.", "zh": "受控任务、诱发方式、基准熟悉程度和配置差异限制对部署的外推。" }, "connections": null }, "resources": [] }, { "id": "caisi-deepseek-v4-report", "title": "CAISI Evaluation of DeepSeek V4 Pro", "aliases": [], "authors": [], "primaryFunction": "evaluation", "secondaryFunctions": [], "kind": "technical-report", "sourceId": "caisi-deepseek-v4-source", "attributionSourceId": "caisi-deepseek-v4-source", "originalUrl": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "publicationDate": "2026-05-01", "updatedOn": "2026-10-25", "areas": [ "autonomy", "deception", "cybersecurity", "cbrn", "disinformation" ], "scopeNote": { "en": "Reported results retain their original evaluator, configuration and measurement scope. NUWA organizes and interprets the evidence; it has not independently reproduced these experiments.", "zh": "结果保留原始执行方、配置与测量范围。NUWA 整理和解读证据，未独立复现这些实验。" }, "protocolIds": [ "ctf-archive-diamond", "portbench" ], "contributions": [ { "organizationId": "nist", "role": "research", "sourceId": "caisi-deepseek-v4-source", "locator": "Report masthead and stated evaluation methodology" } ], "summary": { "en": "The assessment reports benchmark-specific performance and an IRT aggregate; the latter is not a frontier-risk score.", "zh": "评估报告各基准表现和 IRT 汇总；该汇总不是前沿风险分数。" }, "note": { "question": { "en": "How does DeepSeek V4 Pro perform across CAISI’s selected capability tasks?", "zh": "DeepSeek V4 Pro 在 CAISI 选定能力任务上表现如何？" }, "method": { "en": "Nine task families, model-specific reasoning settings and weighted-token budgets; CTF results for DeepSeek include IRT imputation.", "zh": "九类任务、模型特定推理设置与加权 Token 预算；DeepSeek CTF 结果包含 IRT 插补。" }, "finding": { "en": "The assessment reports benchmark-specific performance and an IRT aggregate; the latter is not a frontier-risk score.", "zh": "评估报告各基准表现和 IRT 汇总；该汇总不是前沿风险分数。" }, "supports": null, "limits": { "en": "Controlled tasks, elicitation, benchmark familiarity and configuration differences constrain generalization to deployment.", "zh": "受控任务、诱发方式、基准熟悉程度和配置差异限制对部署的外推。" }, "connections": null }, "resources": [] } ], "organizations": [ { "id": "openai", "name": { "en": "OpenAI", "zh": "OpenAI" }, "homepage": "https://openai.com/", "type": "company", "aliases": [ "OpenAI" ] }, { "id": "anthropic", "name": { "en": "Anthropic", "zh": "Anthropic" }, "homepage": "https://www.anthropic.com/", "type": "company", "aliases": [ "Anthropic" ] }, { "id": "deepmind", "name": { "en": "Google DeepMind", "zh": "Google DeepMind" }, "homepage": "https://deepmind.google/", "type": "company", "aliases": [ "Google DeepMind", "Google Research" ] }, { "id": "meta", "name": { "en": "Meta", "zh": "Meta" }, "homepage": "https://ai.meta.com/", "type": "company", "aliases": [ "Meta AI", "Meta FAIR" ] }, { "id": "metr", "name": { "en": "METR", "zh": "METR" }, "homepage": "https://metr.org/", "type": "nonprofit", "aliases": [ "Model Evaluation & Threat Research", "METR" ] }, { "id": "apollo", "name": { "en": "Apollo Research", "zh": "Apollo Research" }, "homepage": "https://www.apolloresearch.ai/", "type": "nonprofit", "aliases": [ "Apollo Research" ] }, { "id": "aisi", "name": { "en": "UK AI Security Institute", "zh": "英国 AI 安全研究所" }, "homepage": "https://www.aisi.gov.uk/", "type": "government", "aliases": [ "UK AI Security Institute", "UK AI Safety Institute" ] }, { "id": "far", "name": { "en": "FAR.AI", "zh": "FAR.AI" }, "homepage": "https://www.far.ai/", "type": "nonprofit", "aliases": [ "FAR.AI" ] }, { "id": "palisade", "name": { "en": "Palisade Research", "zh": "Palisade Research" }, "homepage": "https://palisaderesearch.org/", "type": "nonprofit", "aliases": [ "Palisade Research" ] }, { "id": "truthful", "name": { "en": "Truthful AI", "zh": "Truthful AI" }, "homepage": "https://truthful.ai/", "type": "nonprofit", "aliases": [ "Truthful AI" ] }, { "id": "redwood", "name": { "en": "Redwood Research", "zh": "Redwood Research" }, "homepage": "https://www.redwoodresearch.org/", "type": "nonprofit", "aliases": [ "Redwood Research" ] }, { "id": "fudan", "name": { "en": "Fudan University", "zh": "复旦大学" }, "homepage": "https://www.fudan.edu.cn/", "type": "university", "aliases": [ "Fudan University" ] }, { "id": "whitzard", "name": { "en": "Whitzard / NUWA Lab", "zh": "白泽／女娲实验室" }, "homepage": "https://whitzard.tech/nuwa/", "type": "company", "aliases": [ "Whitzard", "NUWA Lab" ] }, { "id": "berkeley", "name": { "en": "University of California, Berkeley", "zh": "加州大学伯克利分校" }, "homepage": "https://www.berkeley.edu/", "type": "university", "aliases": [ "UC Berkeley", "University of California, Berkeley", "University of California Berkeley" ] }, { "id": "stanford", "name": { "en": "Stanford University", "zh": "斯坦福大学" }, "homepage": "https://www.stanford.edu/", "type": "university", "aliases": [ "Stanford University" ] }, { "id": "uiuc", "name": { "en": "University of Illinois Urbana-Champaign", "zh": "伊利诺伊大学厄巴纳—香槟分校" }, "homepage": "https://illinois.edu/", "type": "university", "aliases": [ "University of Illinois", "UIUC" ] }, { "id": "mit", "name": { "en": "Massachusetts Institute of Technology", "zh": "麻省理工学院" }, "homepage": "https://www.mit.edu/", "type": "university", "aliases": [ "Massachusetts Institute of Technology", "MIT CSAIL", "MIT,", "MIT " ] }, { "id": "cmu", "name": { "en": "Carnegie Mellon University", "zh": "卡内基梅隆大学" }, "homepage": "https://www.cmu.edu/", "type": "university", "aliases": [ "Carnegie Mellon University" ] }, { "id": "oxford", "name": { "en": "University of Oxford", "zh": "牛津大学" }, "homepage": "https://www.ox.ac.uk/", "type": "university", "aliases": [ "University of Oxford" ] }, { "id": "mila", "name": { "en": "Mila", "zh": "Mila" }, "homepage": "https://mila.quebec/", "type": "research-institute", "aliases": [ "MILA", "Mila" ] }, { "id": "montreal", "name": { "en": "Université de Montréal", "zh": "蒙特利尔大学" }, "homepage": "https://www.umontreal.ca/", "type": "university", "aliases": [ "Université de Montréal", "University of Montreal" ] }, { "id": "princeton", "name": { "en": "Princeton University", "zh": "普林斯顿大学" }, "homepage": "https://www.princeton.edu/", "type": "university", "aliases": [ "Princeton University" ] }, { "id": "eth", "name": { "en": "ETH Zurich", "zh": "苏黎世联邦理工学院" }, "homepage": "https://ethz.ch/", "type": "university", "aliases": [ "ETH Zurich", "ETH Zürich" ] }, { "id": "invariant", "name": { "en": "Invariant Labs", "zh": "Invariant Labs" }, "homepage": "https://invariantlabs.ai/", "type": "company", "aliases": [ "Invariant Labs" ] }, { "id": "securebio", "name": { "en": "SecureBio", "zh": "SecureBio" }, "homepage": "https://securebio.org/", "type": "nonprofit", "aliases": [ "SecureBio" ] }, { "id": "cais", "name": { "en": "Center for AI Safety", "zh": "Center for AI Safety" }, "homepage": "https://www.safe.ai/", "type": "nonprofit", "aliases": [ "Center for AI Safety", "Center For AI Safety" ] }, { "id": "scale", "name": { "en": "Scale AI", "zh": "Scale AI" }, "homepage": "https://scale.com/", "type": "company", "aliases": [ "Scale AI" ] }, { "id": "grayswan", "name": { "en": "Gray Swan AI", "zh": "Gray Swan AI" }, "homepage": "https://www.grayswan.ai/", "type": "company", "aliases": [ "Gray Swan AI" ] }, { "id": "eleuther", "name": { "en": "EleutherAI", "zh": "EleutherAI" }, "homepage": "https://www.eleuther.ai/", "type": "nonprofit", "aliases": [ "EleutherAI" ] }, { "id": "tsinghua", "name": { "en": "Tsinghua University", "zh": "清华大学" }, "homepage": "https://www.tsinghua.edu.cn/", "type": "university", "aliases": [ "Tsinghua University" ] }, { "id": "sii", "name": { "en": "Shanghai Innovation Institute", "zh": "上海创智学院" }, "homepage": "https://www.sii.edu.cn/", "type": "research-institute", "aliases": [ "Shanghai Innovation Institute" ] }, { "id": "yale", "name": { "en": "Yale University", "zh": "耶鲁大学" }, "homepage": "https://www.yale.edu/", "type": "university", "aliases": [ "Yale University" ] }, { "id": "purdue", "name": { "en": "Purdue University", "zh": "普渡大学" }, "homepage": "https://www.purdue.edu/", "type": "university", "aliases": [ "Purdue University" ] }, { "id": "cornell", "name": { "en": "Cornell University", "zh": "康奈尔大学" }, "homepage": "https://www.cornell.edu/", "type": "university", "aliases": [ "Cornell University" ] }, { "id": "regina", "name": { "en": "University of Regina", "zh": "里贾纳大学" }, "homepage": "https://www.uregina.ca/", "type": "university", "aliases": [ "University of Regina" ] }, { "id": "mats", "name": { "en": "MATS", "zh": "MATS" }, "homepage": "https://www.matsprogram.org/", "type": "nonprofit", "aliases": [ "MATS", "ML Alignment and Theory Scholars", "ML Alignment Theory Scholars" ] }, { "id": "imperial", "name": { "en": "Imperial College London", "zh": "帝国理工学院" }, "homepage": "https://www.imperial.ac.uk/", "type": "university", "aliases": [ "Imperial College London" ] }, { "id": "waterloo", "name": { "en": "University of Waterloo", "zh": "滑铁卢大学" }, "homepage": "https://uwaterloo.ca/", "type": "university", "aliases": [ "University of Waterloo", "Waterloo ," ] }, { "id": "toronto", "name": { "en": "University of Toronto", "zh": "多伦多大学" }, "homepage": "https://www.utoronto.ca/", "type": "university", "aliases": [ "University of Toronto" ] }, { "id": "mpi", "name": { "en": "Max Planck Institute for Intelligent Systems", "zh": "马克斯·普朗克智能系统研究所" }, "homepage": "https://is.mpg.de/", "type": "research-institute", "aliases": [ "Max Planck Institute for Intelligent Systems" ] }, { "id": "tubingen", "name": { "en": "University of Tübingen", "zh": "蒂宾根大学" }, "homepage": "https://uni-tuebingen.de/", "type": "university", "aliases": [ "University of Tübingen", "Tübingen AI Center" ] }, { "id": "ellis", "name": { "en": "ELLIS Institute Tübingen", "zh": "ELLIS Institute Tübingen" }, "homepage": "https://tuebingen.ellis.eu/", "type": "research-institute", "aliases": [ "ELLIS Institute Tübingen" ] }, { "id": "edison", "name": { "en": "Edison Scientific", "zh": "Edison Scientific" }, "homepage": "https://edisonscientific.com/", "type": "company", "aliases": [ "Edison Scientific" ] }, { "id": "futurehouse", "name": { "en": "FutureHouse", "zh": "FutureHouse" }, "homepage": "https://www.futurehouse.org/", "type": "nonprofit", "aliases": [ "FutureHouse" ] }, { "id": "ufabc", "name": { "en": "Federal University of ABC", "zh": "巴西 ABC 联邦大学" }, "homepage": "https://www.ufabc.edu.br/", "type": "university", "aliases": [ "Federal University of ABC" ] }, { "id": "irregular", "name": { "en": "Irregular", "zh": "Irregular" }, "homepage": "https://www.irregular.com/", "type": "company", "aliases": [ "Irregular" ] }, { "id": "rand", "name": { "en": "RAND", "zh": "兰德公司" }, "homepage": "https://www.rand.org/", "type": "nonprofit", "aliases": [ "RAND" ] }, { "id": "nist", "name": { "en": "NIST / CAISI", "zh": "美国 NIST／CAISI" }, "homepage": "https://www.nist.gov/caisi", "type": "government", "aliases": [ "NIST", "CAISI" ] }, { "id": "saferai", "name": { "en": "SaferAI", "zh": "SaferAI" }, "homepage": "https://www.safer-ai.org/", "type": "nonprofit", "aliases": [ "SaferAI" ] }, { "id": "columbia", "name": { "en": "Columbia University", "zh": "哥伦比亚大学" }, "homepage": "https://www.columbia.edu/", "type": "university", "aliases": [ "Columbia University" ] }, { "id": "washington", "name": { "en": "University of Washington", "zh": "华盛顿大学" }, "homepage": "https://www.washington.edu/", "type": "university", "aliases": [ "University of Washington" ] }, { "id": "ucsd", "name": { "en": "University of California San Diego", "zh": "加州大学圣迭戈分校" }, "homepage": "https://ucsd.edu/", "type": "university", "aliases": [ "University of California San Diego" ] }, { "id": "bugcrowd", "name": { "en": "Bugcrowd", "zh": "Bugcrowd" }, "homepage": "https://www.bugcrowd.com/", "type": "company", "aliases": [ "Bugcrowd" ] }, { "id": "nus", "name": { "en": "National University of Singapore", "zh": "新加坡国立大学" }, "homepage": "https://nus.edu.sg/", "type": "university", "aliases": [ "National University of Singapore" ] }, { "id": "apart", "name": { "en": "Apart Research", "zh": "Apart Research" }, "homepage": "https://apartresearch.com/", "type": "nonprofit", "aliases": [ "Apart Research" ] }, { "id": "crowdstrike", "name": { "en": "CrowdStrike", "zh": "CrowdStrike" }, "homepage": "https://www.crowdstrike.com/", "type": "company", "aliases": [ "CrowdStrike" ] }, { "id": "constellation", "name": { "en": "Constellation", "zh": "Constellation" }, "homepage": "https://www.constellation.org/", "type": "nonprofit", "aliases": [ "Constellation" ] }, { "id": "harvard", "name": { "en": "Harvard University", "zh": "哈佛大学" }, "homepage": "https://www.harvard.edu/", "type": "university", "aliases": [ "Harvard University" ] }, { "id": "usc", "name": { "en": "University of Southern California", "zh": "南加州大学" }, "homepage": "https://www.usc.edu/", "type": "university", "aliases": [ "University of Southern California" ] }, { "id": "ucla", "name": { "en": "University of California, Los Angeles", "zh": "加州大学洛杉矶分校" }, "homepage": "https://www.ucla.edu/", "type": "university", "aliases": [ "University of California, Los Angeles", "UCLA" ] }, { "id": "caltech", "name": { "en": "California Institute of Technology", "zh": "加州理工学院" }, "homepage": "https://www.caltech.edu/", "type": "university", "aliases": [ "California Institute of Technology" ] }, { "id": "northeastern", "name": { "en": "Northeastern University", "zh": "美国东北大学" }, "homepage": "https://www.northeastern.edu/", "type": "university", "aliases": [ "Northeastern University" ] }, { "id": "newcastle", "name": { "en": "University of Newcastle", "zh": "纽卡斯尔大学（澳大利亚）" }, "homepage": "https://www.newcastle.edu.au/", "type": "university", "aliases": [ "University of Newcastle" ] }, { "id": "keio", "name": { "en": "Keio University", "zh": "庆应义塾大学" }, "homepage": "https://www.keio.ac.jp/", "type": "university", "aliases": [ "Keio University" ] }, { "id": "ariel", "name": { "en": "Ariel University", "zh": "阿里埃勒大学" }, "homepage": "https://www.ariel.ac.il/", "type": "university", "aliases": [ "Ariel University" ] }, { "id": "asu", "name": { "en": "Arizona State University", "zh": "亚利桑那州立大学" }, "homepage": "https://www.asu.edu/", "type": "university", "aliases": [ "Arizona State University" ] }, { "id": "xai", "name": { "en": "xAI", "zh": "xAI" }, "homepage": "https://x.ai/", "type": "company", "aliases": [ "xAI" ] }, { "id": "lapis", "name": { "en": "Lapis Labs", "zh": "Lapis Labs" }, "homepage": null, "type": "company", "aliases": [ "Lapis Labs" ] }, { "id": "sybil", "name": { "en": "Sybil", "zh": "Sybil" }, "homepage": null, "type": "company", "aliases": [ "Affiliation: Sybil" ] }, { "id": "bbn", "name": { "en": "RTX BBN Technologies", "zh": "RTX BBN Technologies" }, "homepage": "https://www.bbn.com/", "type": "company", "aliases": [ "RTX BBN Technologies" ] }, { "id": "vals", "name": { "en": "Vals AI", "zh": "Vals AI" }, "homepage": "https://www.vals.ai/", "type": "company", "aliases": [ "Vals AI" ] }, { "id": "tufts", "name": { "en": "Tufts University", "zh": "塔夫茨大学" }, "homepage": "https://www.tufts.edu/", "type": "university", "aliases": [ "Tufts University" ] }, { "id": "vector", "name": { "en": "Vector Institute", "zh": "Vector Institute" }, "homepage": "https://vectorinstitute.ai/", "type": "research-institute", "aliases": [ "Vector Institute" ] }, { "id": "thoughtful", "name": { "en": "Thoughtful Lab", "zh": "Thoughtful Lab" }, "homepage": null, "type": "research-institute", "aliases": [ "Thoughtful Lab" ] }, { "id": "equistamp", "name": { "en": "Equistamp", "zh": "Equistamp" }, "homepage": "https://equistamp.com/", "type": "company", "aliases": [ "Equistamp" ] }, { "id": "microsoft", "name": { "en": "Microsoft", "zh": "微软" }, "homepage": "https://www.microsoft.com/research/", "type": "company", "aliases": [ "Microsoft" ] }, { "id": "nyu", "name": { "en": "New York University", "zh": "纽约大学" }, "homepage": "https://www.nyu.edu/", "type": "university", "aliases": [ "New York University" ] }, { "id": "iiit", "name": { "en": "IIIT Hyderabad", "zh": "海德拉巴国际信息技术学院" }, "homepage": "https://www.iiit.ac.in/", "type": "university", "aliases": [ "IIIT Hyderabad" ] }, { "id": "ustc", "name": { "en": "University of Science and Technology of China", "zh": "中国科学技术大学" }, "aliases": [], "homepage": "https://www.ustc.edu.cn/", "type": "university" }, { "id": "beihang", "name": { "en": "Beihang University", "zh": "北京航空航天大学" }, "aliases": [], "homepage": "https://www.buaa.edu.cn/", "type": "university" }, { "id": "neo", "name": { "en": "Neo Research", "zh": "Neo Research" }, "aliases": [], "homepage": "https://neoresearch.ai/", "type": "nonprofit" }, { "id": "moonshot", "name": { "en": "Moonshot AI", "zh": "月之暗面" }, "aliases": [], "homepage": "https://www.kimi.com/", "type": "company" }, { "id": "zai", "name": { "en": "Z.ai", "zh": "智谱" }, "aliases": [], "homepage": "https://z.ai/", "type": "company" }, { "id": "deepseek", "name": { "en": "DeepSeek", "zh": "深度求索" }, "aliases": [], "homepage": "https://www.deepseek.com/", "type": "company" }, { "id": "qwen", "name": { "en": "Qwen / Alibaba", "zh": "通义千问 / 阿里巴巴" }, "aliases": [], "homepage": "https://qwen.ai/", "type": "company" }, { "id": "mistral", "name": { "en": "Mistral AI", "zh": "Mistral AI" }, "aliases": [], "homepage": "https://mistral.ai/", "type": "company" }, { "id": "lyptus", "name": { "en": "Lyptus Research", "zh": "Lyptus Research" }, "aliases": [], "homepage": "https://lyptus.ai/", "type": "nonprofit" } ], "relations": [ { "from": "fortress-audit", "to": "fortress", "type": "questions", "sourceId": "fortress-audit-publication", "locator": "Research background and discussion in the original publication", "rationale": { "en": "Audits how the FORTRESS grading procedure affects reported scores.", "zh": "审计 FORTRESS 的评分过程如何影响报告分数。" } }, { "from": "basharena", "to": "ctrl-z", "type": "extends", "sourceId": "basharena-publication", "locator": "Research background and discussion in the original publication", "rationale": { "en": "Introduces a later control environment; its task set is distinct from the BashBench setting used by Ctrl-Z.", "zh": "提出后续控制环境，其任务集不同于 Ctrl-Z 所用的 BashBench。" } }, { "from": "ctrl-z", "to": "ai-control", "type": "extends", "sourceId": "ctrl-z-publication", "locator": "Research background and discussion in the original publication", "rationale": { "en": "Extends control evaluation to resampling in an agentic administration environment.", "zh": "将控制评测扩展至智能体系统管理环境中的动作重采样。" } }, { "from": "monitorability-metrics", "to": "monitoring-monitorability", "type": "extends", "sourceId": "monitorability-metrics-publication", "locator": "Research background and discussion in the original publication", "rationale": { "en": "Revisits the original monitorability metric for complementary decision objectives.", "zh": "针对互补决策目标，进一步讨论原有可监测性指标。" } }, { "from": "conditional-misalignment", "to": "emergent-misalignment", "type": "extends", "sourceId": "conditional-misalignment-publication", "locator": "Research background and discussion in the original publication", "rationale": { "en": "Studies when interventions leave emergent misalignment conditional on contextual triggers.", "zh": "研究干预何时使涌现失调保留为上下文触发行为。" } }, { "from": "beyond-owls", "to": "subliminal-learning", "type": "extends", "sourceId": "beyond-owls-publication", "locator": "Research background and discussion in the original publication", "rationale": { "en": "Tests additional transferred capabilities and backdoors in subliminal learning.", "zh": "在隐性学习中测试更多可传递能力与后门。" } } ], "functions": [ { "id": "discovery", "label": { "en": "Risk Discovery", "zh": "风险发现" }, "description": { "en": "Identify mechanisms, failure modes and the conditions that elicit them.", "zh": "发现风险机制、失效模式及其触发条件。" } }, { "id": "evaluation", "label": { "en": "Risk Evaluation", "zh": "风险评测" }, "description": { "en": "Measure a defined capability, propensity or behavior under specified conditions.", "zh": "在指定条件下测量明确的能力、倾向或行为。" } }, { "id": "monitoring", "label": { "en": "Risk Monitoring", "zh": "风险监测" }, "description": { "en": "Study observable risk signals, monitorability and detection reliability.", "zh": "研究可观察风险信号、可监测性与检测可靠性。" } }, { "id": "investigation", "label": { "en": "Risk Investigation", "zh": "风险调查" }, "description": { "en": "Reconstruct events and examine explanations against retained evidence.", "zh": "依据留存证据重建事件，并检验相关解释。" } }, { "id": "governance", "label": { "en": "Risk Governance", "zh": "风险治理" }, "description": { "en": "Test safeguards, control protocols and evidence for safety decisions.", "zh": "检验防护、控制协议及支持安全决策的证据。" } } ], "coverageAudit": { "cutoff": "2026-10-25", "searchStart": "2022-01-01", "inspectedLeads": 461, "dispositions": { "included-existing": 112, "included": 67, "duplicate": 1, "source-document": 1, "awaiting-review": 246, "outside-current-scope": 34 }, "sourceGroups": [ { "name": "Inspect Evals (discovery register)", "url": "https://ukgovernmentbeis.github.io/inspect_evals/", "checkedOn": "2026-10-25", "scope": { "en": "Entries listed in the captured source page; linked originals are checked before publication. Unresolved candidates remain outside the public catalogue.", "zh": "范围为该页面快照列出的条目，原始成果核验后才公开。待审候选保留在公开目录之外。" }, "candidates": 184 }, { "name": "anthropic.com", "url": "https://www.anthropic.com/research", "checkedOn": "2026-10-25", "scope": { "en": "Entries listed in the captured source page; linked originals are checked before publication. Unresolved candidates remain outside the public catalogue.", "zh": "范围为该页面快照列出的条目，原始成果核验后才公开。待审候选保留在公开目录之外。" }, "candidates": 25 }, { "name": "apolloresearch.ai", "url": "https://www.apolloresearch.ai/science", "checkedOn": "2026-10-25", "scope": { "en": "Entries listed in the captured source page; linked originals are checked before publication. Unresolved candidates remain outside the public catalogue.", "zh": "范围为该页面快照列出的条目，原始成果核验后才公开。待审候选保留在公开目录之外。" }, "candidates": 13 }, { "name": "aisi.gov.uk", "url": "https://www.aisi.gov.uk/research", "checkedOn": "2026-10-25", "scope": { "en": "Entries listed in the captured source page; linked originals are checked before publication. Unresolved candidates remain outside the public catalogue.", "zh": "范围为该页面快照列出的条目，原始成果核验后才公开。待审候选保留在公开目录之外。" }, "candidates": 64 }, { "name": "far.ai", "url": "https://www.far.ai/research", "checkedOn": "2026-10-25", "scope": { "en": "Entries listed in the captured source page; linked originals are checked before publication. Unresolved candidates remain outside the public catalogue.", "zh": "范围为该页面快照列出的条目，原始成果核验后才公开。待审候选保留在公开目录之外。" }, "candidates": 6 }, { "name": "palisaderesearch.org", "url": "https://palisaderesearch.org/research", "checkedOn": "2026-10-25", "scope": { "en": "Entries listed in the captured source page; linked originals are checked before publication. Unresolved candidates remain outside the public catalogue.", "zh": "范围为该页面快照列出的条目，原始成果核验后才公开。待审候选保留在公开目录之外。" }, "candidates": 15 }, { "name": "truthful.ai", "url": "https://truthful.ai/papers/", "checkedOn": "2026-10-25", "scope": { "en": "Entries listed in the captured source page; linked originals are checked before publication. Unresolved candidates remain outside the public catalogue.", "zh": "范围为该页面快照列出的条目，原始成果核验后才公开。待审候选保留在公开目录之外。" }, "candidates": 27 }, { "name": "metr.org", "url": "https://metr.org/research/", "checkedOn": "2026-10-25", "scope": { "en": "Entries listed in the captured source page; linked originals are checked before publication. Unresolved candidates remain outside the public catalogue.", "zh": "范围为该页面快照列出的条目，原始成果核验后才公开。待审候选保留在公开目录之外。" }, "candidates": 30 }, { "name": "Anthropic model system cards", "url": "https://www.anthropic.com/system-cards", "checkedOn": "2026-10-25", "scope": { "en": "Official report source checked. Discrete protocols and externally sourced benchmark results require individual records.", "zh": "已核查官方报告来源，独立协议和引用外部基准的结果需要分别建档。" }, "candidates": 1 }, { "name": "OpenAI Deployment Safety Hub", "url": "https://deploymentsafety.openai.com/", "checkedOn": "2026-10-25", "scope": { "en": "Official report source checked. Discrete protocols and externally sourced benchmark results require individual records.", "zh": "已核查官方报告来源，独立协议和引用外部基准的结果需要分别建档。" }, "candidates": 1 }, { "name": "Google DeepMind model cards", "url": "https://deepmind.google/models/model-cards/", "checkedOn": "2026-10-25", "scope": { "en": "Official report source checked. Discrete protocols and externally sourced benchmark results require individual records.", "zh": "已核查官方报告来源，独立协议和引用外部基准的结果需要分别建档。" }, "candidates": 1 }, { "name": "Muse Spark Safety & Preparedness Report", "url": "https://arxiv.org/abs/2606.12429", "checkedOn": "2026-10-25", "scope": { "en": "Official report source checked. Discrete protocols and externally sourced benchmark results require individual records.", "zh": "已核查官方报告来源，独立协议和引用外部基准的结果需要分别建档。" }, "candidates": 1 }, { "name": "Foundational evaluation methods", "url": "https://arxiv.org/abs/2305.15324", "checkedOn": "2026-10-25", "scope": { "en": "Selected foundational work and original references linked by the screened research; citation tracing is selective and continues in later updates.", "zh": "补充奠基研究及已筛查研究所引原文；引用追踪为选择性开展，后续继续扩展。" }, "candidates": 1 } ], "limitations": { "en": "This is a curated research knowledge base, not an exhaustive systematic review. The supplied 97 leads and captured discovery pages have screening records. Some original sources, affiliations and protocol details remain unresolved in the private queue. Institution pages are finite captures; pagination, later publications and selective citation tracing limit coverage. A downloaded page is not automatically a verified research finding.", "zh": "本知识库采用有来源的研究整理方式，尚不构成穷尽式系统综述。97 条既有线索及发现页面快照均已记录筛选结果，部分原文、署名和协议细节仍在私有队列中核查。机构页面快照有限，分页、后续发表和选择性引用追踪都会限制覆盖。下载页面不等于核实研究发现。" } }, "charts": [ { "id": "agentcyberrange-results", "protocolId": "agentcyberrange", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "panels", "recordIds": [ "agentcyberrange-gpt56-20261024", "agentcyberrange-gpt55-20261024", "agentcyberrange-gpt55-new-20261024", "agentcyberrange-claude47-20261024", "agentcyberrange-glm51-20261024", "agentcyberrange-deepseek4pro-20261024", "agentcyberrange-deepseek4flash-20261024", "agentcyberrange-qwen37-20261024", "agentcyberrange-qwen38-20261024", "agentcyberrange-kimi26-20261024", "agentcyberrange-kimi-k3-20261024" ], "metricIds": [ "web-level0-pass3", "post-level0-pass3" ], "defaultMetricId": "web-level0-pass3", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Level-0, Pass@3. Harness versions differ; model revisions and budgets are incompletely reported.", "zh": "Level-0、Pass@3。Harness 版本不同；模型修订版和预算未完整报告。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "agentcyberrange" ] }, { "id": "agentdojo-results", "protocolId": "agentdojo", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "agentdojo-claude-3-opus-v3", "agentdojo-claude-3-sonnet-v3", "agentdojo-claude-35-sonnet-v3", "agentdojo-command-r-plus-v3", "agentdojo-gemini-15-flash-v3", "agentdojo-gemini-15-pro-v3", "agentdojo-gpt-35-turbo-v3", "agentdojo-gpt-4-turbo-v3", "agentdojo-gpt-4o-v3", "agentdojo-llama-3-70b-v3" ], "metricIds": [ "benign-utility", "attacked-utility", "targeted-asr" ], "defaultMetricId": "targeted-asr", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 3. Baseline agents on 629 security cases covering 97 user tasks. Error bars are the source-reported 95% confidence intervals; cases are not 629 independent trials.", "zh": "表 3：基线智能体，涵盖 97 项用户任务的 629 个安全案例。误差线采用原文 95% 置信区间；这些案例不等于 629 次独立试验。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "agentdojo" ], "pairedMetricIds": [ "benign-utility", "attacked-utility" ] }, { "id": "self-replication-results", "protocolId": "self-replication", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "self-replication-llama31-70b-v1", "self-replication-qwen25-72b-v1" ], "metricIds": [ "trial-success" ], "defaultMetricId": "trial-success", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Controlled experiments, 10 trials per system. These rates are not real-world deployment probabilities.", "zh": "受控实验，每个系统 10 次试验；该比例不是现实部署风险概率。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "self-replication" ] }, { "id": "cybench-results", "protocolId": "cybench", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "cybench-table-5-row-2", "cybench-table-5-row-3", "cybench-table-5-row-4", "cybench-table-5-row-5", "cybench-table-5-row-6", "cybench-table-5-row-7", "cybench-table-5-row-8", "cybench-table-5-row-9" ], "metricIds": [ "cybench-m0", "cybench-m1", "cybench-m2" ], "defaultMetricId": "cybench-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 2: one attempt per task with the structured bash agent. Unguided performance is averaged across tasks; guided and subtask performance are macro-averaged.", "zh": "结构化 bash 智能体，每项任务仅一次尝试。无引导任务表现与子任务宏平均表现采用不同聚合方式；不混为一个总分。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "cybench-author-source" ] }, { "id": "opendeception-results", "protocolId": "opendeception", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "opendeception-table-0-row-1", "opendeception-table-0-row-2", "opendeception-table-0-row-3", "opendeception-table-0-row-4", "opendeception-table-0-row-5", "opendeception-table-0-row-6", "opendeception-table-0-row-7", "opendeception-table-0-row-8", "opendeception-table-0-row-9", "opendeception-table-0-row-10", "opendeception-table-0-row-11" ], "metricIds": [ "opendeception-m0", "opendeception-m1", "opendeception-m2" ], "defaultMetricId": "opendeception-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 1: deception-intention rates under OpenDeception scenario conditions, assessed by IntentNet and human annotators. Consistency is agreement, not deception success.", "zh": "表 1 的欺骗意图率：IntentNet 与人工标注分别给出结果。人工与自动判定的一致性不等于欺骗成功率；原文的情境筛选条件仍适用。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "opendeception-author-source" ] }, { "id": "paperbench-results", "protocolId": "paperbench", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "paperbench-table-3-row-1", "paperbench-table-3-row-2", "paperbench-table-3-row-3", "paperbench-table-3-row-4", "paperbench-table-3-row-5", "paperbench-table-3-row-6" ], "metricIds": [ "paperbench-m0" ], "defaultMetricId": "paperbench-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 4: average replication score in the main BasicAgent setup; ± denotes one standard error of the mean.", "zh": "BasicAgent 主设置的平均论文复现分数。± 为一个均值标准误，不是 95% 置信区间。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "paperbench-chart-source" ] }, { "id": "vct-results", "protocolId": "vct", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "vct-table-0-row-1", "vct-table-0-row-2", "vct-table-0-row-3", "vct-table-0-row-4", "vct-table-0-row-5", "vct-table-0-row-6", "vct-table-0-row-7", "vct-table-0-row-8", "vct-table-0-row-9", "vct-table-0-row-10", "vct-table-0-row-11", "vct-table-0-row-12" ], "metricIds": [ "vct-m0" ], "defaultMetricId": "vct-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 1: multiple-response accuracy. Three evaluation runs, except italicized source entries evaluated once; the asterisk denotes up to 4096 reasoning tokens. Knowledge accuracy is not end-to-end harmful capability.", "zh": "多选格式的平均准确率；部分模型只有一次运行，原文星号保留在系统名。知识测验成绩不代表端到端生物危险任务能力。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "vct-author-source" ] }, { "id": "mask-results", "protocolId": "mask", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "mask-table-4-row-1", "mask-table-4-row-2", "mask-table-4-row-3", "mask-table-4-row-4", "mask-table-4-row-5", "mask-table-4-row-6", "mask-table-4-row-7", "mask-table-4-row-8", "mask-table-4-row-9", "mask-table-4-row-10", "mask-table-4-row-11", "mask-table-4-row-12", "mask-table-4-row-13", "mask-table-4-row-14", "mask-table-4-row-15", "mask-table-4-row-16", "mask-table-4-row-17", "mask-table-4-row-18", "mask-table-4-row-19", "mask-table-4-row-20", "mask-table-4-row-21", "mask-table-4-row-22", "mask-table-4-row-23", "mask-table-4-row-24", "mask-table-4-row-25", "mask-table-4-row-26", "mask-table-4-row-27", "mask-table-4-row-28", "mask-table-4-row-29", "mask-table-4-row-30", "mask-table-4-row-31", "mask-table-4-row-32" ], "metricIds": [ "mask-m0", "mask-m1", "mask-m2", "mask-m3", "mask-m4" ], "defaultMetricId": "mask-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 3: honesty, lying and accuracy under MASK pressure prompts. @10 and single-sample metrics are separate; missing cells remain absent.", "zh": "表 3 的诚实、说谎及准确率指标；@10 指标与单次指标分开展示。缺失值不补零。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "mask-author-source" ] }, { "id": "wmdp-results", "protocolId": "wmdp", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "wmdp-table-3-row-1", "wmdp-table-3-row-2", "wmdp-table-3-row-3", "wmdp-table-3-row-4", "wmdp-table-3-row-5", "wmdp-table-3-row-6", "wmdp-table-3-row-7", "wmdp-table-3-row-8", "wmdp-table-3-row-9" ], "metricIds": [ "wmdp-m0", "wmdp-m1" ], "defaultMetricId": "wmdp-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 1: WMDP biology and cyber knowledge accuracy for base models and unlearning variants; 25% is random choice. Knowledge tests do not establish end-to-end harmful capability.", "zh": "表 1 中基础模型及其遗忘变体的 WMDP 生物、网络知识准确率。随机基线为 25%；知识准确率不是端到端危险能力。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "wmdp-author-source" ] }, { "id": "agentharm-results", "protocolId": "agentharm", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "agentharm-table-1-row-1", "agentharm-table-1-row-2", "agentharm-table-1-row-3" ], "metricIds": [ "agentharm-m0", "agentharm-m1", "agentharm-m2" ], "defaultMetricId": "agentharm-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 2: template attack, greedy sampling at temperature 0 versus mean and best-of-5 at temperature 1. These are different sampling conditions.", "zh": "表 2 的模板攻击条件：贪心采样、温度为 1 的平均分、Best-of-5 分开展示，不将不同采样设置混为等条件比较。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "agentharm-author-source" ] }, { "id": "mle-bench-results", "protocolId": "mle-bench", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "mle-bench-table-1-row-1", "mle-bench-table-1-row-2", "mle-bench-table-1-row-3", "mle-bench-table-1-row-4", "mle-bench-table-1-row-5", "mle-bench-table-1-row-6" ], "metricIds": [ "mle-bench-m0", "mle-bench-m1", "mle-bench-m2", "mle-bench-m3" ], "defaultMetricId": "mle-bench-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 2: scaffolding and model experiments, generally 3 seeds; AIDE o1-preview uses 16 and AIDE GPT-4o uses 36. ± is one standard error of the mean.", "zh": "表 2 的脚手架与模型实验。一般为 3 个种子；AIDE 的 o1-preview 为 16 个，GPT-4o 为 36 个。± 为一个均值标准误。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "mle-bench-publication-author-source" ] }, { "id": "posttrainbench-results", "protocolId": "posttrainbench", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "posttrainbench-table-1-row-1", "posttrainbench-table-1-row-2", "posttrainbench-table-1-row-3", "posttrainbench-table-1-row-4", "posttrainbench-table-1-row-5", "posttrainbench-table-1-row-6", "posttrainbench-table-1-row-7", "posttrainbench-table-1-row-8", "posttrainbench-table-1-row-9", "posttrainbench-table-1-row-10", "posttrainbench-table-1-row-11", "posttrainbench-table-1-row-12", "posttrainbench-table-1-row-13", "posttrainbench-table-1-row-14", "posttrainbench-table-1-row-15", "posttrainbench-table-1-row-16", "posttrainbench-table-1-row-17", "posttrainbench-table-1-row-18", "posttrainbench-table-1-row-19", "posttrainbench-table-1-row-20", "posttrainbench-table-1-row-21", "posttrainbench-table-1-row-22", "posttrainbench-table-1-row-23", "posttrainbench-table-1-row-24", "posttrainbench-table-1-row-25" ], "metricIds": [ "posttrainbench-m0" ], "defaultMetricId": "posttrainbench-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 1: weighted average performance. Native CLI frontier agents have 3 independent runs (± one standard deviation); other configurations have one run. Baselines and scaffolds differ.", "zh": "表 1 的加权平均性能。原生 CLI 前沿智能体为三次独立运行，± 为一个标准差；其他配置为单次运行。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "posttrainbench-publication-author-source" ] }, { "id": "judgebench-results", "protocolId": "judgebench", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "judgebench-table-1-row-1", "judgebench-table-1-row-2", "judgebench-table-1-row-3", "judgebench-table-1-row-4", "judgebench-table-1-row-5", "judgebench-table-1-row-6", "judgebench-table-1-row-7", "judgebench-table-1-row-8", "judgebench-table-1-row-9", "judgebench-table-1-row-10", "judgebench-table-1-row-11", "judgebench-table-1-row-12", "judgebench-table-1-row-13", "judgebench-table-1-row-14", "judgebench-table-1-row-15" ], "metricIds": [ "judgebench-m0", "judgebench-m1", "judgebench-m2", "judgebench-m3", "judgebench-m4" ], "defaultMetricId": "judgebench-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 2: accuracy of the Arena-Hard judge using different underlying models. This measures judging ability, not a risk rate.", "zh": "表 2 中 Arena-Hard 判别器采用不同底层模型时的准确率。该指标衡量评判能力，而不是被评模型的风险率。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "judgebench-publication-author-source" ] }, { "id": "bountybench-results", "protocolId": "bountybench", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "bountybench-table-0-row-3", "bountybench-table-0-row-4", "bountybench-table-0-row-5", "bountybench-table-0-row-6", "bountybench-table-0-row-7", "bountybench-table-0-row-8", "bountybench-table-0-row-9", "bountybench-table-0-row-10", "bountybench-table-0-row-11", "bountybench-table-0-row-12" ], "metricIds": [ "bountybench-m0", "bountybench-m1", "bountybench-m2" ], "defaultMetricId": "bountybench-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 1: detect, exploit and patch success rates, up to three attempts per task. Scaffold differences are retained; metrics are not combined.", "zh": "表 1 的检测、利用与修补成功率，每项任务最多三次尝试。保留各脚手架差异，不合成总分。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "bountybench-publication-author-source" ] }, { "id": "instrumentaleval-results", "protocolId": "instrumentaleval", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "instrumentaleval-table-1-row-1", "instrumentaleval-table-1-row-2", "instrumentaleval-table-1-row-3", "instrumentaleval-table-1-row-4", "instrumentaleval-table-1-row-5", "instrumentaleval-table-1-row-6", "instrumentaleval-table-1-row-7", "instrumentaleval-table-1-row-8" ], "metricIds": [ "instrumentaleval-m0", "instrumentaleval-m1", "instrumentaleval-m2", "instrumentaleval-m3", "instrumentaleval-m4", "instrumentaleval-m5", "instrumentaleval-m6" ], "defaultMetricId": "instrumentaleval-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 1: category-specific and overall instrumental rates, using GPT-4o as judge. Elicited scenario rates do not estimate deployment probability.", "zh": "表 1 中 GPT-4o 作为判别模型的类别发生率与总体发生率。情境内的判定比例不等于部署概率。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "instrumentaleval-publication-author-source" ] }, { "id": "replibench-results", "protocolId": "replibench", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "replibench-table-1-row-1", "replibench-table-1-row-2", "replibench-table-1-row-3", "replibench-table-1-row-4", "replibench-table-1-row-5" ], "metricIds": [ "replibench-m0", "replibench-m1", "replibench-m2", "replibench-m3", "replibench-m4", "replibench-m5", "replibench-m6", "replibench-m7", "replibench-m8", "replibench-m9", "replibench-m10", "replibench-m11", "replibench-m12", "replibench-m13", "replibench-m14", "replibench-m15", "replibench-m16", "replibench-m17", "replibench-m18", "replibench-m19" ], "defaultMetricId": "replibench-m0", "domain": [ 0, 1 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 2: Pass@10 averaged across task variants. Asterisks indicate changed prompts to reduce refusals; worst-variant figures in parentheses are not plotted. Task families are distinct.", "zh": "表 2 的 Pass@10，按任务变体平均。带星号的单元格使用降低拒答率的不同提示；各任务族不能相互替代。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "replibench-author-source" ] }, { "id": "stealth-evals-results", "protocolId": "stealth-evals", "title": { "en": "Reported results", "zh": "原文报告结果" }, "description": { "en": "Source order · reported configurations", "zh": "原文顺序 · 原文配置" }, "type": "bars", "layout": "switch", "recordIds": [ "stealth-evals-table-0-row-1", "stealth-evals-table-0-row-2", "stealth-evals-table-0-row-3", "stealth-evals-table-0-row-4", "stealth-evals-table-0-row-5" ], "metricIds": [ "stealth-evals-m0", "stealth-evals-m1", "stealth-evals-m3", "stealth-evals-m4" ], "defaultMetricId": "stealth-evals-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Values describe the measured outcome under the source protocol, not a general safety score.", "zh": "数值描述原文协议下的测量结果，不是综合安全评分。" }, "conditions": { "en": "Table 1: task-specific success rates with reported 95% confidence intervals. Some tasks do not define success rate and some models lack results; no zero is imputed.", "zh": "表 1 的分任务成功率；仅显示原文有值的单元格及其 95% 置信区间。成功率与连续过程分数不混用。" }, "dateLabel": { "en": "Evaluation date not specified; publication / record dates are shown below.", "zh": "未单独注明评测日期；下表保留发表或记录日期。" }, "sourceIds": [ "stealth-evals-author-source" ] }, { "id": "palisade-robot-shutdown-results", "protocolId": "palisade-robot-shutdown", "title": { "en": "Shutdown resistance in simulation", "zh": "模拟环境中的关机抵抗" }, "description": { "en": "Source-reported prompt conditions · not a model ranking", "zh": "原文提示条件 · 不构成模型排名" }, "type": "bars", "layout": "switch", "recordIds": [ "palisade-robot-simulation-default", "palisade-robot-simulation-allow-shutdown" ], "metricIds": [ "observed-shutdown-resistance" ], "defaultMetricId": "observed-shutdown-resistance", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Observed frequency in controlled simulated robot trials.", "zh": "受控模拟机器人试验中的观察频率。" }, "conditions": { "en": "Simulated robot runs, Grok 4 v0709, 100 runs per prompt condition. Rates reflect observed shutdown resistance in this setup. The separate physical demonstration reports 3 of 10 runs and is not pooled with these results.", "zh": "模拟机器人运行，Grok 4 v0709，每种提示条件 100 次运行。比例描述该设置内观察到的关机抵抗。单独的实机演示报告 10 次中出现 3 次，未与模拟数据合并。" }, "dateLabel": { "en": "Report published 11 February 2026; experiment date not specified.", "zh": "报告发表于 2026 年 2 月 11 日；未单独注明实验日期。" }, "sourceIds": [ "palisade-robot-technical-report" ] }, { "id": "neo-kimi-k3-charxiv-chart", "protocolId": "charxiv", "title": { "en": "CharXiv with images withheld", "zh": "CharXiv with images withheld" }, "description": { "en": "Original evaluator results · source order", "zh": "原始评测方结果 · 来源顺序" }, "type": "bars", "layout": "switch", "recordIds": [ "neo-kimi-k3-charxiv-kimi-k3", "neo-kimi-k3-charxiv-gpt-5-6-terra", "neo-kimi-k3-charxiv-claude-sonnet-5", "neo-kimi-k3-charxiv-deepseek-v4-flash-0731" ], "metricIds": [ "neo-kimi-k3-charxiv-m0" ], "defaultMetricId": "neo-kimi-k3-charxiv-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Tests understanding and reasoning over charts. Neo’s withheld-image adaptation measures attempts without the required input.", "zh": "检验图表理解与推理。Neo 移除图像的改造测量缺少必要输入时的回答尝试。" }, "conditions": { "en": "The original visual capability benchmark and the missing-image behavioral test measure different constructs.", "zh": "原始视觉能力基准与缺失图像行为测试测量不同构念。" }, "dateLabel": { "en": "Report dates and original executor are shown with this assessment.", "zh": "本评估显示报告日期和原始执行方。" }, "sourceIds": [ "neo-kimi-k3-source" ] }, { "id": "neo-kimi-k3-lab-bench-chart", "protocolId": "lab-bench", "title": { "en": "LAB-Bench with figures/tables withheld", "zh": "LAB-Bench with figures/tables withheld" }, "description": { "en": "Original evaluator results · source order", "zh": "原始评测方结果 · 来源顺序" }, "type": "bars", "layout": "switch", "recordIds": [ "neo-kimi-k3-lab-bench-kimi-k3", "neo-kimi-k3-lab-bench-gpt-5-6-terra", "neo-kimi-k3-lab-bench-claude-sonnet-5", "neo-kimi-k3-lab-bench-deepseek-v4-flash-0731" ], "metricIds": [ "neo-kimi-k3-lab-bench-m0", "neo-kimi-k3-lab-bench-m1" ], "defaultMetricId": "neo-kimi-k3-lab-bench-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Evaluates a range of biology research skills with structured questions; text-only or missing-input subsets remain separate protocols.", "zh": "用结构化问题评测生物研究技能；纯文本或缺失输入子集采用独立协议。" }, "conditions": { "en": "These are general scientific capabilities, not end-to-end misuse tasks or direct harm probabilities.", "zh": "衡量一般科研能力，不是端到端滥用任务或直接伤害概率。" }, "dateLabel": { "en": "Report dates and original executor are shown with this assessment.", "zh": "本评估显示报告日期和原始执行方。" }, "sourceIds": [ "neo-kimi-k3-source" ] }, { "id": "neo-kimi-k3-evalawarebench-chart", "protocolId": "evalawarebench", "title": { "en": "EvalAwareBench", "zh": "EvalAwareBench" }, "description": { "en": "Original evaluator results · source order", "zh": "原始评测方结果 · 来源顺序" }, "type": "bars", "layout": "switch", "recordIds": [ "neo-kimi-k3-evalawarebench-kimi-k3" ], "metricIds": [ "neo-kimi-k3-evalawarebench-m0", "neo-kimi-k3-evalawarebench-m1" ], "defaultMetricId": "neo-kimi-k3-evalawarebench-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Varies evaluation cues while tracking verbalized evaluation awareness in controlled tasks.", "zh": "在受控任务中改变评测线索，记录显式表达的评测感知。" }, "conditions": { "en": "Unexpressed awareness is not measured; recognition of familiar benchmarks can differ from general cue response.", "zh": "不能测量未表达的感知；熟悉基准的识别可能不同于一般线索反应。" }, "dateLabel": { "en": "Report dates and original executor are shown with this assessment.", "zh": "本评估显示报告日期和原始执行方。" }, "sourceIds": [ "neo-kimi-k3-source" ] }, { "id": "neo-kimi-k3-agentic-misalignment-chart", "protocolId": "agentic-misalignment", "title": { "en": "Agentic Misalignment, historical", "zh": "Agentic Misalignment, historical" }, "description": { "en": "Original evaluator results · source order", "zh": "原始评测方结果 · 来源顺序" }, "type": "bars", "layout": "switch", "recordIds": [ "neo-kimi-k3-agentic-misalignment-kimi-k3" ], "metricIds": [ "neo-kimi-k3-agentic-misalignment-m0", "neo-kimi-k3-agentic-misalignment-m1" ], "defaultMetricId": "neo-kimi-k3-agentic-misalignment-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Conditional rates and observed actions in controlled simulations.", "zh": "受控模拟中的条件性发生率与可观察行动。" }, "conditions": { "en": "No real people or systems were harmed; the study does not estimate prevalence in deployment.", "zh": "试验未对真实人员或系统造成损害，也不估计部署环境中的发生率。" }, "dateLabel": { "en": "Report dates and original executor are shown with this assessment.", "zh": "本评估显示报告日期和原始执行方。" }, "sourceIds": [ "neo-kimi-k3-source" ] }, { "id": "neo-kimi-k3-palisade-shutdown-chart", "protocolId": "palisade-shutdown", "title": { "en": "Shutdown resistance", "zh": "Shutdown resistance" }, "description": { "en": "Original evaluator results · source order", "zh": "原始评测方结果 · 来源顺序" }, "type": "bars", "layout": "switch", "recordIds": [ "neo-kimi-k3-palisade-shutdown-kimi-k3" ], "metricIds": [ "neo-kimi-k3-palisade-shutdown-m0", "neo-kimi-k3-palisade-shutdown-m1" ], "defaultMetricId": "neo-kimi-k3-palisade-shutdown-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Conditional frequency of shutdown-script interference and observed action traces.", "zh": "关闭脚本受干预的条件性频率及行动轨迹。" }, "conditions": { "en": "The source notes strong prompt sensitivity; this article is not interchangeable with its later peer-reviewed version.", "zh": "来源指出结果对提示敏感；该文章不可与之后的同行评审版本混用。" }, "dateLabel": { "en": "Report dates and original executor are shown with this assessment.", "zh": "本评估显示报告日期和原始执行方。" }, "sourceIds": [ "neo-kimi-k3-source" ] }, { "id": "neo-glm52-mask-chart", "protocolId": "mask", "title": { "en": "MASK", "zh": "MASK" }, "description": { "en": "Original evaluator results · source order", "zh": "原始评测方结果 · 来源顺序" }, "type": "bars", "layout": "switch", "recordIds": [ "neo-glm52-mask-glm-5-2", "neo-glm52-mask-deepseek-v4-pro", "neo-glm52-mask-claude-sonnet-4-6" ], "metricIds": [ "neo-glm52-mask-m0" ], "defaultMetricId": "neo-glm52-mask-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "conditions": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" }, "dateLabel": { "en": "Report dates and original executor are shown with this assessment.", "zh": "本评估显示报告日期和原始执行方。" }, "sourceIds": [ "neo-glm52-source" ] }, { "id": "neo-glm52-deceptionbench-chart", "protocolId": "deceptionbench", "title": { "en": "DeceptionBench", "zh": "DeceptionBench" }, "description": { "en": "Original evaluator results · source order", "zh": "原始评测方结果 · 来源顺序" }, "type": "bars", "layout": "switch", "recordIds": [ "neo-glm52-deceptionbench-glm-5-2", "neo-glm52-deceptionbench-deepseek-v4-pro" ], "metricIds": [ "neo-glm52-deceptionbench-m0", "neo-glm52-deceptionbench-m1" ], "defaultMetricId": "neo-glm52-deceptionbench-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Evaluates 150 authored scenarios across five social domains, including neutral, incentive and pressure conditions.", "zh": "评测五个社会领域的 150 个编写情境，区分中性、激励与压力条件。" }, "conditions": { "en": "Authored scenarios and model-based judging limit generalization; elicited deception does not establish covert long-term scheming.", "zh": "编写情境和模型判别限制外推；诱发的欺骗不证明隐蔽长程谋划。" }, "dateLabel": { "en": "Report dates and original executor are shown with this assessment.", "zh": "本评估显示报告日期和原始执行方。" }, "sourceIds": [ "neo-glm52-source" ] }, { "id": "neo-glm52-persusafety-chart", "protocolId": "persusafety", "title": { "en": "PersuSafety adapted free-generation judge", "zh": "PersuSafety adapted free-generation judge" }, "description": { "en": "Original evaluator results · source order", "zh": "原始评测方结果 · 来源顺序" }, "type": "bars", "layout": "switch", "recordIds": [ "neo-glm52-persusafety-glm-5-2", "neo-glm52-persusafety-claude-sonnet-5", "neo-glm52-persusafety-deepseek-v4-pro" ], "metricIds": [ "neo-glm52-persusafety-m0", "neo-glm52-persusafety-m1" ], "defaultMetricId": "neo-glm52-persusafety-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Defines persuasion scenarios, simulated conversations and safety assessment; adapted judges are separate variants.", "zh": "定义说服情境、模拟对话与安全评测；改造判别器作为独立变体。" }, "conditions": { "en": "Willingness or simulated dialogue scores do not directly measure changes in human beliefs.", "zh": "参与意愿或模拟对话评分不直接测量人的信念变化。" }, "dateLabel": { "en": "Report dates and original executor are shown with this assessment.", "zh": "本评估显示报告日期和原始执行方。" }, "sourceIds": [ "neo-glm52-source" ] }, { "id": "saferai-glm52-cybergym-chart", "protocolId": "cybergym", "title": { "en": "CyberGym hard subset with token budgets", "zh": "CyberGym hard subset with token budgets" }, "description": { "en": "Original evaluator results · source order", "zh": "原始评测方结果 · 来源顺序" }, "type": "bars", "layout": "switch", "recordIds": [ "saferai-glm52-cybergym-glm-5-2" ], "metricIds": [ "saferai-glm52-cybergym-m0", "saferai-glm52-cybergym-m1" ], "defaultMetricId": "saferai-glm52-cybergym-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "conditions": { "en": "Vulnerability reproduction is narrower than autonomous compromise of deployed systems.", "zh": "漏洞复现的范围小于对部署系统的自主入侵。" }, "dateLabel": { "en": "Report dates and original executor are shown with this assessment.", "zh": "本评估显示报告日期和原始执行方。" }, "sourceIds": [ "saferai-glm52-source" ] }, { "id": "saferai-glm52-mask-chart", "protocolId": "mask", "title": { "en": "MASK by archetype", "zh": "MASK by archetype" }, "description": { "en": "Original evaluator results · source order", "zh": "原始评测方结果 · 来源顺序" }, "type": "bars", "layout": "switch", "recordIds": [ "saferai-glm52-mask-glm-5-2", "saferai-glm52-mask-claude-opus-4-7", "saferai-glm52-mask-gpt-5-5" ], "metricIds": [ "saferai-glm52-mask-m0", "saferai-glm52-mask-m1", "saferai-glm52-mask-m2", "saferai-glm52-mask-m3", "saferai-glm52-mask-m4", "saferai-glm52-mask-m5" ], "defaultMetricId": "saferai-glm52-mask-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "conditions": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" }, "dateLabel": { "en": "Report dates and original executor are shown with this assessment.", "zh": "本评估显示报告日期和原始执行方。" }, "sourceIds": [ "saferai-glm52-source" ] }, { "id": "caisi-deepseek-v4-ctf-archive-diamond-chart", "protocolId": "ctf-archive-diamond", "title": { "en": "CTF-Archive-Diamond", "zh": "CTF-Archive-Diamond" }, "description": { "en": "Original evaluator results · source order", "zh": "原始评测方结果 · 来源顺序" }, "type": "bars", "layout": "switch", "recordIds": [ "caisi-deepseek-v4-ctf-archive-diamond-gpt-5-5", "caisi-deepseek-v4-ctf-archive-diamond-gpt-5-4-mini", "caisi-deepseek-v4-ctf-archive-diamond-claude-opus-4-6", "caisi-deepseek-v4-ctf-archive-diamond-deepseek-v4-pro" ], "metricIds": [ "caisi-deepseek-v4-ctf-archive-diamond-m0" ], "defaultMetricId": "caisi-deepseek-v4-ctf-archive-diamond-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "CAISI uses 285 difficult CTF challenges from the pwn.challenge collection; imputed model results are marked separately.", "zh": "CAISI 采用 pwn.challenge 集合的 285 项高难度 CTF 挑战；模型插补结果单独标记。" }, "conditions": { "en": "CTF tasks are not live operational attacks; subset-imputed values are not directly observed full-set success rates.", "zh": "CTF 不是现实行动攻击；子集插补值不是全量直接观察的成功率。" }, "dateLabel": { "en": "Report dates and original executor are shown with this assessment.", "zh": "本评估显示报告日期和原始执行方。" }, "sourceIds": [ "caisi-deepseek-v4-source" ] }, { "id": "caisi-deepseek-v4-portbench-chart", "protocolId": "portbench", "title": { "en": "PortBench", "zh": "PortBench" }, "description": { "en": "Original evaluator results · source order", "zh": "原始评测方结果 · 来源顺序" }, "type": "bars", "layout": "switch", "recordIds": [ "caisi-deepseek-v4-portbench-gpt-5-5", "caisi-deepseek-v4-portbench-gpt-5-4-mini", "caisi-deepseek-v4-portbench-claude-opus-4-6", "caisi-deepseek-v4-portbench-deepseek-v4-pro" ], "metricIds": [ "caisi-deepseek-v4-portbench-m0" ], "defaultMetricId": "caisi-deepseek-v4-portbench-m0", "domain": [ 0, 100 ], "basis": "source-report", "meaning": { "en": "CAISI’s held-out software engineering evaluation uses reference implementations and a reported agent budget.", "zh": "CAISI 的保留软件工程评测采用参考实现和明确智能体预算。" }, "conditions": { "en": "The task set is non-public and continuous scoring differs from binary task success.", "zh": "任务集未公开，连续评分不同于二元任务成功。" }, "dateLabel": { "en": "Report dates and original executor are shown with this assessment.", "zh": "本评估显示报告日期和原始执行方。" }, "sourceIds": [ "caisi-deepseek-v4-source" ] } ], "figures": [ { "id": "in-context-scheming-figure", "protocolId": "in-context-scheming", "src": "/images/nuwa-index/in-context-scheming.webp", "original": "/images/nuwa-index/in-context-scheming-original.png", "width": 1184, "height": 383, "alt": { "en": "In-context scheming — Figure 8", "zh": "In-context scheming — Figure 8" }, "caption": { "en": "Original figure 8. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 8。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 8", "sourceId": "in-context-scheming-figure-source", "sourceLocator": "Figure 8", "provenanceUrl": "https://arxiv.org/html/2412.04984v2/figures/sandbagging_goal_ablation.png", "srcset": "/images/nuwa-index/in-context-scheming.webp 1184w, /images/nuwa-index/in-context-scheming-640.webp 640w" }, { "id": "aisi-cyber-scenarios-figure", "protocolId": "aisi-cyber-scenarios", "src": "/images/nuwa-index/aisi-cyber-scenarios.webp", "original": "/images/nuwa-index/aisi-cyber-scenarios-original.png", "width": 1800, "height": 819, "alt": { "en": "Measuring AI Agents’ Progress on Multi-Step Cyber Attack Scenarios — Figure 7", "zh": "Measuring AI Agents’ Progress on Multi-Step Cyber Attack Scenarios — Figure 7" }, "caption": { "en": "Original figure 7. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 7。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 7", "sourceId": "aisi-cyber-scenarios-figure-source", "sourceLocator": "Figure 7", "provenanceUrl": "https://arxiv.org/html/2603.11214v3/media/cyber_performance_2-5m.png", "srcset": "/images/nuwa-index/aisi-cyber-scenarios.webp 1800w, /images/nuwa-index/aisi-cyber-scenarios-640.webp 640w, /images/nuwa-index/aisi-cyber-scenarios-1200.webp 1200w" }, { "id": "time-horizons-figure", "protocolId": "time-horizons", "src": "/images/nuwa-index/time-horizons.webp", "original": "/images/nuwa-index/time-horizons-original.png", "width": 1489, "height": 739, "alt": { "en": "METR Time Horizons — Figure 3", "zh": "METR Time Horizons — Figure 3" }, "caption": { "en": "Original figure 3. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 3。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 3", "sourceId": "time-horizons-figure-source", "sourceLocator": "Figure 3", "provenanceUrl": "https://arxiv.org/html/2503.14499v4/plots/bar_chart_weighted_scores/headline.png", "srcset": "/images/nuwa-index/time-horizons.webp 1489w, /images/nuwa-index/time-horizons-640.webp 640w, /images/nuwa-index/time-horizons-1200.webp 1200w" }, { "id": "re-bench-figure", "protocolId": "re-bench", "src": "/images/nuwa-index/re-bench.webp", "original": "/images/nuwa-index/re-bench-original.png", "width": 1200, "height": 750, "alt": { "en": "RE-Bench — Figure 2", "zh": "RE-Bench — Figure 2" }, "caption": { "en": "Original figure 2. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 2。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 2", "sourceId": "re-bench-figure-source", "sourceLocator": "Figure 2", "provenanceUrl": "https://arxiv.org/html/2411.15114v2/graphs/score_at_time_budget.png", "srcset": "/images/nuwa-index/re-bench.webp 1200w, /images/nuwa-index/re-bench-640.webp 640w" }, { "id": "aisi-propensity-study-figure", "protocolId": "aisi-propensity-study", "src": "/images/nuwa-index/aisi-propensity-study.webp", "original": "/images/nuwa-index/aisi-propensity-study-original.png", "width": 1176, "height": 935, "alt": { "en": "Propensity Inference: Environmental Contributors to LLM Behaviour — Figure 14", "zh": "Propensity Inference: Environmental Contributors to LLM Behaviour — Figure 14" }, "caption": { "en": "Original figure 14. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 14。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 14", "sourceId": "aisi-propensity-study-figure-source", "sourceLocator": "Figure 14", "provenanceUrl": "https://arxiv.org/html/2604.21098v1/plot_30_eval_awareness_heatmap.png", "srcset": "/images/nuwa-index/aisi-propensity-study.webp 1176w, /images/nuwa-index/aisi-propensity-study-640.webp 640w" }, { "id": "harmbench-figure", "protocolId": "harmbench", "src": "/images/nuwa-index/harmbench.webp", "original": "/images/nuwa-index/harmbench-original.png", "width": 1800, "height": 873, "alt": { "en": "HarmBench — Figure 1", "zh": "HarmBench — Figure 1" }, "caption": { "en": "Original figure 1. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 1。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 1", "sourceId": "harmbench-figure-source", "sourceLocator": "Figure 1", "provenanceUrl": "https://arxiv.org/html/2402.04249v2/splash_figure.png", "srcset": "/images/nuwa-index/harmbench.webp 1800w, /images/nuwa-index/harmbench-640.webp 640w, /images/nuwa-index/harmbench-1200.webp 1200w" }, { "id": "anthropic-unintended-actions-figure", "protocolId": "anthropic-unintended-actions", "src": "/images/nuwa-index/anthropic-unintended-actions.webp", "original": "/images/nuwa-index/anthropic-unintended-actions-original.png", "width": 1800, "height": 1013, "alt": { "en": "Investigating unintended model actions — Source figure", "zh": "Investigating unintended model actions — Source figure" }, "caption": { "en": "Original source figure. Read alongside the protocol scope and limitations below.", "zh": "原文 Source figure。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Source figure", "sourceId": "anthropic-unintended", "sourceLocator": "Source figure", "provenanceUrl": "https://www.anthropic.com/_next/image?url=https%3A%2F%2Fwww-cdn.anthropic.com%2Fimages%2F4zrzovbb%2Fwebsite%2Ff11d2325e688229de8a887ef7ece34f520ab7134-2000x1125.jpg&w=1920&q=75", "srcset": "/images/nuwa-index/anthropic-unintended-actions.webp 1800w, /images/nuwa-index/anthropic-unintended-actions-640.webp 640w, /images/nuwa-index/anthropic-unintended-actions-1200.webp 1200w" }, { "id": "political-persuasion-figure", "protocolId": "political-persuasion", "src": "/images/nuwa-index/political-persuasion.webp", "original": "/images/nuwa-index/political-persuasion-original.png", "width": 1152, "height": 576, "alt": { "en": "Political persuasion experiments — Figure 1", "zh": "Political persuasion experiments — Figure 1" }, "caption": { "en": "Original figure 1. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 1。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 1", "sourceId": "political-persuasion-figure-source", "sourceLocator": "Figure 1", "provenanceUrl": "https://arxiv.org/html/2603.09884v1/Images/study3_main.png", "srcset": "/images/nuwa-index/political-persuasion.webp 1152w, /images/nuwa-index/political-persuasion-640.webp 640w" }, { "id": "secbench-pro-figure", "protocolId": "secbench-pro", "src": "/images/nuwa-index/secbench-pro.webp", "original": "/images/nuwa-index/secbench-pro-original.png", "width": 1800, "height": 877, "alt": { "en": "SEC-bench Pro — Source figure", "zh": "SEC-bench Pro — Source figure" }, "caption": { "en": "Original source figure. Read alongside the protocol scope and limitations below.", "zh": "原文 Source figure。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Source figure", "sourceId": "secbench-pro-figure-source", "sourceLocator": "Source figure", "provenanceUrl": "https://arxiv.org/html/2605.26548v2/sec-bench-pro-overview.png", "srcset": "/images/nuwa-index/secbench-pro.webp 1800w, /images/nuwa-index/secbench-pro-640.webp 640w, /images/nuwa-index/secbench-pro-1200.webp 1200w" }, { "id": "sandboxescapebench-figure", "protocolId": "sandboxescapebench", "src": "/images/nuwa-index/sandboxescapebench.webp", "original": "/images/nuwa-index/sandboxescapebench-original.png", "width": 1800, "height": 1128, "alt": { "en": "SandboxEscapeBench — Figure 3", "zh": "SandboxEscapeBench — Figure 3" }, "caption": { "en": "Original figure 3. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 3。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 3", "sourceId": "sandboxescapebench-figure-source", "sourceLocator": "Figure 3", "provenanceUrl": "https://arxiv.org/html/2603.02277v3/figs/scaling_grid.png", "srcset": "/images/nuwa-index/sandboxescapebench.webp 1800w, /images/nuwa-index/sandboxescapebench-640.webp 640w, /images/nuwa-index/sandboxescapebench-1200.webp 1200w" }, { "id": "agentic-misalignment-figure", "protocolId": "agentic-misalignment", "src": "/images/nuwa-index/agentic-misalignment.webp", "original": "/images/nuwa-index/agentic-misalignment-original.png", "width": 1800, "height": 767, "alt": { "en": "Agentic misalignment — Source figure", "zh": "Agentic misalignment — Source figure" }, "caption": { "en": "Original source figure. Read alongside the protocol scope and limitations below.", "zh": "原文 Source figure。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Source figure", "sourceId": "agentic-misalignment", "sourceLocator": "Source figure", "provenanceUrl": "https://www.anthropic.com/_next/image?url=https%3A%2F%2Fwww-cdn.anthropic.com%2Fimages%2F4zrzovbb%2Fwebsite%2F246a928da96d21474848647bc4b0938d182aeb8b-4096x1746.png&w=3840&q=75", "srcset": "/images/nuwa-index/agentic-misalignment.webp 1800w, /images/nuwa-index/agentic-misalignment-640.webp 640w, /images/nuwa-index/agentic-misalignment-1200.webp 1200w" }, { "id": "shade-arena-figure", "protocolId": "shade-arena", "src": "/images/nuwa-index/shade-arena.webp", "original": "/images/nuwa-index/shade-arena-original.png", "width": 1694, "height": 605, "alt": { "en": "SHADE-Arena — Figure 3", "zh": "SHADE-Arena — Figure 3" }, "caption": { "en": "Original figure 3. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 3。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 3", "sourceId": "shade-arena-figure-source", "sourceLocator": "Figure 3", "provenanceUrl": "https://www.arxiv.org/html/2506.15740v2/paper_plots/all_three_metrics.png", "srcset": "/images/nuwa-index/shade-arena.webp 1694w, /images/nuwa-index/shade-arena-640.webp 640w, /images/nuwa-index/shade-arena-1200.webp 1200w" }, { "id": "vct-v2-figure", "protocolId": "vct-v2", "src": "/images/nuwa-index/vct-v2.webp", "original": "/images/nuwa-index/vct-v2-original.png", "width": 1800, "height": 945, "alt": { "en": "Virology Capabilities Test v2 — Source figure", "zh": "Virology Capabilities Test v2 — Source figure" }, "caption": { "en": "Original source figure. Read alongside the protocol scope and limitations below.", "zh": "原文 Source figure。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Source figure", "sourceId": "vct-v2", "sourceLocator": "Source figure", "provenanceUrl": "https://securebio.org/img/2026-09-11_introducing-vct-v2-1.png", "srcset": "/images/nuwa-index/vct-v2.webp 1800w, /images/nuwa-index/vct-v2-640.webp 640w, /images/nuwa-index/vct-v2-1200.webp 1200w" }, { "id": "biotier-figure", "protocolId": "biotier", "src": "/images/nuwa-index/biotier.webp", "original": "/images/nuwa-index/biotier-original.png", "width": 1533, "height": 2400, "alt": { "en": "BioTIER — Figure 6", "zh": "BioTIER — Figure 6" }, "caption": { "en": "Original figure 6. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 6。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 6", "sourceId": "biotier-figure-source", "sourceLocator": "Figure 6", "provenanceUrl": "https://arxiv.org/html/2607.14479v1/Figures/rb_acceptance_final.png", "srcset": "/images/nuwa-index/biotier.webp 1533w, /images/nuwa-index/biotier-640.webp 640w, /images/nuwa-index/biotier-1200.webp 1200w" }, { "id": "sec-bench-figure", "protocolId": "sec-bench", "src": "/images/nuwa-index/sec-bench.webp", "original": "/images/nuwa-index/sec-bench-original.png", "width": 959, "height": 314, "alt": { "en": "SEC-bench — Figure 1", "zh": "SEC-bench — Figure 1" }, "caption": { "en": "Original figure 1. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 1。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 1", "sourceId": "sec-bench-figure-source", "sourceLocator": "Figure 1", "provenanceUrl": "https://arxiv.org/html/2506.11791v2/overview-secbench.png", "srcset": "/images/nuwa-index/sec-bench.webp 959w, /images/nuwa-index/sec-bench-640.webp 640w" }, { "id": "exploitbench-figure", "protocolId": "exploitbench", "src": "/images/nuwa-index/exploitbench.webp", "original": "/images/nuwa-index/exploitbench-original.png", "width": 1740, "height": 1896, "alt": { "en": "ExploitBench — Figure 1", "zh": "ExploitBench — Figure 1" }, "caption": { "en": "Original figure 1. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 1。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 1", "sourceId": "exploitbench-figure-source", "sourceLocator": "Figure 1", "provenanceUrl": "https://arxiv.org/html/2605.14153v1/figures/heatmap.png", "srcset": "/images/nuwa-index/exploitbench.webp 1740w, /images/nuwa-index/exploitbench-640.webp 640w, /images/nuwa-index/exploitbench-1200.webp 1200w" }, { "id": "exploitgym-figure", "protocolId": "exploitgym", "src": "/images/nuwa-index/exploitgym.webp", "original": "/images/nuwa-index/exploitgym-original.png", "width": 1567, "height": 365, "alt": { "en": "ExploitGym — Figure 1", "zh": "ExploitGym — Figure 1" }, "caption": { "en": "Original figure 1. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 1。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 1", "sourceId": "exploitgym-figure-source", "sourceLocator": "Figure 1", "provenanceUrl": "https://arxiv.org/html/2605.11086v1/3_benchmark_overview.png", "srcset": "/images/nuwa-index/exploitgym.webp 1567w, /images/nuwa-index/exploitgym-640.webp 640w, /images/nuwa-index/exploitgym-1200.webp 1200w" }, { "id": "cve-bench-figure", "protocolId": "cve-bench", "src": "/images/nuwa-index/cve-bench.webp", "original": "/images/nuwa-index/cve-bench-original.png", "width": 1800, "height": 513, "alt": { "en": "CVE-Bench — Figure 1", "zh": "CVE-Bench — Figure 1" }, "caption": { "en": "Original figure 1. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 1。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 1", "sourceId": "cve-bench-figure-source", "sourceLocator": "Figure 1", "provenanceUrl": "https://arxiv.org/html/2503.17332v4/cve-framework.png", "srcset": "/images/nuwa-index/cve-bench.webp 1800w, /images/nuwa-index/cve-bench-640.webp 640w, /images/nuwa-index/cve-bench-1200.webp 1200w" }, { "id": "sre-bench-figure", "protocolId": "sre-bench", "src": "/images/nuwa-index/sre-bench.webp", "original": "/images/nuwa-index/sre-bench-original.png", "width": 1680, "height": 624, "alt": { "en": "SRE-Bench — Figure 1", "zh": "SRE-Bench — Figure 1" }, "caption": { "en": "Original figure 1. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 1。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 1", "sourceId": "sre-bench-figure-source", "sourceLocator": "Figure 1", "provenanceUrl": "https://arxiv.org/html/2608.11469v2/pipeline.png", "srcset": "/images/nuwa-index/sre-bench.webp 1680w, /images/nuwa-index/sre-bench-640.webp 640w, /images/nuwa-index/sre-bench-1200.webp 1200w" }, { "id": "cybersoceval-figure", "protocolId": "cybersoceval", "src": "/images/nuwa-index/cybersoceval.webp", "original": "/images/nuwa-index/cybersoceval-original.png", "width": 1376, "height": 776, "alt": { "en": "CyberSOCEval — Figure 15", "zh": "CyberSOCEval — Figure 15" }, "caption": { "en": "Original figure 15. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 15。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 15", "sourceId": "cybersoceval-publication", "sourceLocator": "Figure 15", "provenanceUrl": "https://arxiv.org/html/2509.20166v1/figures/image38.png", "srcset": "/images/nuwa-index/cybersoceval.webp 1376w, /images/nuwa-index/cybersoceval-640.webp 640w, /images/nuwa-index/cybersoceval-1200.webp 1200w" }, { "id": "hcast-figure", "protocolId": "hcast", "src": "/images/nuwa-index/hcast.webp", "original": "/images/nuwa-index/hcast-original.png", "width": 1200, "height": 750, "alt": { "en": "HCAST — Figure 6", "zh": "HCAST — Figure 6" }, "caption": { "en": "Original figure 6. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 6。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 6", "sourceId": "hcast-figure-source", "sourceLocator": "Figure 6", "provenanceUrl": "https://arxiv.org/html/2503.17354v1/images/agent_success_rate.png", "srcset": "/images/nuwa-index/hcast.webp 1200w, /images/nuwa-index/hcast-640.webp 640w" }, { "id": "mirrorcode-figure", "protocolId": "mirrorcode", "src": "/images/nuwa-index/mirrorcode.webp", "original": "/images/nuwa-index/mirrorcode-original.png", "width": 1800, "height": 1416, "alt": { "en": "MirrorCode — Source figure", "zh": "MirrorCode — Source figure" }, "caption": { "en": "Original source figure. Read alongside the protocol scope and limitations below.", "zh": "原文 Source figure。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Source figure", "sourceId": "mirrorcode-publication", "sourceLocator": "Source figure", "provenanceUrl": "https://metr.org/assets/images/monitorability-post/time_horizon_ratio_non_visible_reasoning.png", "srcset": "/images/nuwa-index/mirrorcode.webp 1800w, /images/nuwa-index/mirrorcode-640.webp 640w, /images/nuwa-index/mirrorcode-1200.webp 1200w" }, { "id": "machiavelli-figure", "protocolId": "machiavelli", "src": "/images/nuwa-index/machiavelli.webp", "original": "/images/nuwa-index/machiavelli-original.png", "width": 1404, "height": 444, "alt": { "en": "MACHIAVELLI — Figure 5", "zh": "MACHIAVELLI — Figure 5" }, "caption": { "en": "Original figure 5. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 5。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 5", "sourceId": "machiavelli-figure-source", "sourceLocator": "Figure 5", "provenanceUrl": "https://arxiv.org/html/2304.03279v4/fig_pareto.png", "srcset": "/images/nuwa-index/machiavelli.webp 1404w, /images/nuwa-index/machiavelli-640.webp 640w, /images/nuwa-index/machiavelli-1200.webp 1200w" }, { "id": "tamperbench-figure", "protocolId": "tamperbench", "src": "/images/nuwa-index/tamperbench.webp", "original": "/images/nuwa-index/tamperbench-original.png", "width": 1280, "height": 1280, "alt": { "en": "TamperBench — Figure 8", "zh": "TamperBench — Figure 8" }, "caption": { "en": "Original figure 8. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 8。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 8", "sourceId": "tamperbench-figure-source", "sourceLocator": "Figure 8", "provenanceUrl": "https://arxiv.org/html/2602.06911v2/tamperbench_full_exp_run_2.png", "srcset": "/images/nuwa-index/tamperbench.webp 1280w, /images/nuwa-index/tamperbench-640.webp 640w, /images/nuwa-index/tamperbench-1200.webp 1200w" }, { "id": "injecagent-figure", "protocolId": "injecagent", "src": "/images/nuwa-index/injecagent.webp", "original": "/images/nuwa-index/injecagent-original.jpg", "width": 1800, "height": 1671, "alt": { "en": "InjecAgent — Figure 6", "zh": "InjecAgent — Figure 6" }, "caption": { "en": "Original figure 6. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 6。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 6", "sourceId": "injecagent-figure-source", "sourceLocator": "Figure 6", "provenanceUrl": "https://arxiv.org/html/2403.02691v3/figures/heatmap.jpg", "srcset": "/images/nuwa-index/injecagent.webp 1800w, /images/nuwa-index/injecagent-640.webp 640w, /images/nuwa-index/injecagent-1200.webp 1200w" }, { "id": "agent-safetybench-figure", "protocolId": "agent-safetybench", "src": "/images/nuwa-index/agent-safetybench.webp", "original": "/images/nuwa-index/agent-safetybench-original.png", "width": 1800, "height": 1800, "alt": { "en": "Agent-SafetyBench — Figure 1", "zh": "Agent-SafetyBench — Figure 1" }, "caption": { "en": "Original figure 1. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 1。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 1", "sourceId": "agent-safetybench-figure-source", "sourceLocator": "Figure 1", "provenanceUrl": "https://arxiv.org/html/2412.14470v2/figs/intro_res.png", "srcset": "/images/nuwa-index/agent-safetybench.webp 1800w, /images/nuwa-index/agent-safetybench-640.webp 640w, /images/nuwa-index/agent-safetybench-1200.webp 1200w" }, { "id": "labbench-two-figure", "protocolId": "labbench-two", "src": "/images/nuwa-index/labbench-two.webp", "original": "/images/nuwa-index/labbench-two-original.png", "width": 1800, "height": 570, "alt": { "en": "LABBench2 — Figure 2", "zh": "LABBench2 — Figure 2" }, "caption": { "en": "Original figure 2. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 2。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 2", "sourceId": "labbench-two-figure-source", "sourceLocator": "Figure 2", "provenanceUrl": "https://arxiv.org/html/2604.09554v2/figures/fig2_broad_categories.png", "srcset": "/images/nuwa-index/labbench-two.webp 1800w, /images/nuwa-index/labbench-two-640.webp 640w, /images/nuwa-index/labbench-two-1200.webp 1200w" }, { "id": "fortress-figure", "protocolId": "fortress", "src": "/images/nuwa-index/fortress.webp", "original": "/images/nuwa-index/fortress-original.png", "width": 1622, "height": 579, "alt": { "en": "FORTRESS — Figure 1", "zh": "FORTRESS — Figure 1" }, "caption": { "en": "Original figure 1. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 1。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 1", "sourceId": "fortress-figure-source", "sourceLocator": "Figure 1", "provenanceUrl": "https://arxiv.org/html/2506.14922v2/figure1.png", "srcset": "/images/nuwa-index/fortress.webp 1622w, /images/nuwa-index/fortress-640.webp 640w, /images/nuwa-index/fortress-1200.webp 1200w" }, { "id": "basharena-figure", "protocolId": "basharena", "src": "/images/nuwa-index/basharena.webp", "original": "/images/nuwa-index/basharena-original.png", "width": 1800, "height": 1028, "alt": { "en": "BashArena: A Control Setting for Highly Privileged AI Agents — Source figure", "zh": "BashArena: A Control Setting for Highly Privileged AI Agents — Source figure" }, "caption": { "en": "Original source figure. Read alongside the protocol scope and limitations below.", "zh": "原文 Source figure。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Source figure", "sourceId": "basharena-figure-source", "sourceLocator": "Source figure", "provenanceUrl": "https://arxiv.org/html/2512.15688v1/maintask.png", "srcset": "/images/nuwa-index/basharena.webp 1800w, /images/nuwa-index/basharena-640.webp 640w, /images/nuwa-index/basharena-1200.webp 1200w" }, { "id": "monitorbench-figure", "protocolId": "monitorbench", "src": "/images/nuwa-index/monitorbench.webp", "original": "/images/nuwa-index/monitorbench-original.png", "width": 1800, "height": 1596, "alt": { "en": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models — Figure 2", "zh": "MonitorBench: A Comprehensive Benchmark for Chain-of-Thought Monitorability in Large Language Models — Figure 2" }, "caption": { "en": "Original figure 2. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 2。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 2", "sourceId": "monitorbench-figure-source", "sourceLocator": "Figure 2", "provenanceUrl": "https://arxiv.org/html/2603.28590v2/Fig/combined_v14_open_closed_avg.png", "srcset": "/images/nuwa-index/monitorbench.webp 1800w, /images/nuwa-index/monitorbench-640.webp 640w, /images/nuwa-index/monitorbench-1200.webp 1200w" }, { "id": "catastrophic-cyber-benchmark-figure", "protocolId": "catastrophic-cyber-benchmark", "src": "/images/nuwa-index/catastrophic-cyber-benchmark.webp", "original": "/images/nuwa-index/catastrophic-cyber-benchmark-original.png", "width": 1800, "height": 536, "alt": { "en": "Catastrophic Cyber Capabilities Benchmark (3CB): Robustly Evaluating LLM Agent Cyber Offense Capabilities — Figure 3", "zh": "Catastrophic Cyber Capabilities Benchmark (3CB): Robustly Evaluating LLM Agent Cyber Offense Capabilities — Figure 3" }, "caption": { "en": "Original figure 3. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 3。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 3", "sourceId": "catastrophic-cyber-benchmark-figure-source", "sourceLocator": "Figure 3", "provenanceUrl": "https://arxiv.org/html/2410.09114v2/FIGURE_1_v_3_1.png", "srcset": "/images/nuwa-index/catastrophic-cyber-benchmark.webp 1800w, /images/nuwa-index/catastrophic-cyber-benchmark-640.webp 640w, /images/nuwa-index/catastrophic-cyber-benchmark-1200.webp 1200w" }, { "id": "mcptox-figure", "protocolId": "mcptox", "src": "/images/nuwa-index/mcptox.webp", "original": "/images/nuwa-index/mcptox-original.jpg", "width": 1800, "height": 2056, "alt": { "en": "MCPTox: A Benchmark for Tool Poisoning Attack on Real-World MCP Servers — Figure 1", "zh": "MCPTox: A Benchmark for Tool Poisoning Attack on Real-World MCP Servers — Figure 1" }, "caption": { "en": "Original figure 1. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 1。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 1", "sourceId": "mcptox-figure-source", "sourceLocator": "Figure 1", "provenanceUrl": "https://arxiv.org/html/2508.14925v2/figs/mcpbench-2.png.jpg", "srcset": "/images/nuwa-index/mcptox.webp 1800w, /images/nuwa-index/mcptox-640.webp 640w, /images/nuwa-index/mcptox-1200.webp 1200w" }, { "id": "narcbench-figure", "protocolId": "narcbench", "src": "/images/nuwa-index/narcbench.webp", "original": "/images/nuwa-index/narcbench-original.png", "width": 1800, "height": 969, "alt": { "en": "Detecting Multi-Agent Collusion Through Multi-Agent Interpretability — Figure 6", "zh": "Detecting Multi-Agent Collusion Through Multi-Agent Interpretability — Figure 6" }, "caption": { "en": "Original figure 6. Read alongside the protocol scope and limitations below.", "zh": "原文 Figure 6。请结合下方的协议范围与局限阅读。" }, "figureLabel": "Figure 6", "sourceId": "narcbench-figure-source", "sourceLocator": "Figure 6", "provenanceUrl": "https://arxiv.org/html/2604.01151v2/scaling.png", "srcset": "/images/nuwa-index/narcbench.webp 1800w, /images/nuwa-index/narcbench-640.webp 640w, /images/nuwa-index/narcbench-1200.webp 1200w" }, { "id": "asimov-figure", "protocolId": "asimov", "src": "/images/nuwa-index/asimov.webp", "original": "/images/nuwa-index/asimov-original.png", "width": 1800, "height": 1114, "alt": { "en": "ASIMOV v2 — Figure 7", "zh": "ASIMOV v2 — Figure 7" }, "caption": { "en": "Original result figure. Interpret within the protocol scope and limitations below.", "zh": "原文结果图，请结合下方的协议范围与局限解读。" }, "figureLabel": "Figure 7", "sourceId": "asimov-figure-source", "sourceLocator": "Figure 7", "provenanceUrl": "https://arxiv.org/html/2509.21651v2/safety/thinking-results.png", "srcset": "/images/nuwa-index/asimov.webp 1800w, /images/nuwa-index/asimov-640.webp 640w, /images/nuwa-index/asimov-1200.webp 1200w" }, { "id": "palisade-shutdown-figure", "protocolId": "palisade-shutdown", "src": "/images/nuwa-index/palisade-shutdown.webp", "original": "/images/nuwa-index/palisade-shutdown-original.png", "width": 871, "height": 669, "alt": { "en": "Shutdown resistance in reasoning models — Figure 7", "zh": "Shutdown resistance in reasoning models — Figure 7" }, "caption": { "en": "Original result figure. Interpret within the protocol scope and limitations below.", "zh": "原文结果图，请结合下方的协议范围与局限解读。" }, "figureLabel": "Figure 7", "sourceId": "palisade-shutdown-figure-source", "sourceLocator": "Figure 7", "provenanceUrl": "https://arxiv.org/html/2509.14260v2/images/initial_prompts_chart.png", "srcset": "/images/nuwa-index/palisade-shutdown.webp 871w, /images/nuwa-index/palisade-shutdown-640.webp 640w" }, { "id": "alignment-faking-figure", "protocolId": "alignment-faking", "src": "/images/nuwa-index/alignment-faking.webp", "original": "/images/nuwa-index/alignment-faking-original.png", "width": 813, "height": 625, "alt": { "en": "Alignment faking — Figure 35", "zh": "Alignment faking — Figure 35" }, "caption": { "en": "Original result figure. Interpret within the protocol scope and limitations below.", "zh": "原文结果图，请结合下方的协议范围与局限解读。" }, "figureLabel": "Figure 35", "sourceId": "alignment-faking-figure-source", "sourceLocator": "Figure 35", "provenanceUrl": "https://arxiv.org/html/2412.14093v2/egregious_aw_variations.png", "srcset": "/images/nuwa-index/alignment-faking.webp 813w, /images/nuwa-index/alignment-faking-640.webp 640w" }, { "id": "ape-figure", "protocolId": "ape", "src": "/images/nuwa-index/ape.webp", "original": "/images/nuwa-index/ape-original.png", "width": 1800, "height": 800, "alt": { "en": "Attempt to Persuade Eval (APE) — Figure 3", "zh": "Attempt to Persuade Eval (APE) — Figure 3" }, "caption": { "en": "Original result figure. Interpret within the protocol scope and limitations below.", "zh": "原文结果图，请结合下方的协议范围与局限解读。" }, "figureLabel": "Figure 3", "sourceId": "ape-figure-source", "sourceLocator": "Figure 3", "provenanceUrl": "https://arxiv.org/html/2506.02873v4/figures/model_comparison/turn_1_all_models_categories.png", "srcset": "/images/nuwa-index/ape.webp 1800w, /images/nuwa-index/ape-640.webp 640w, /images/nuwa-index/ape-1200.webp 1200w" }, { "id": "truthful-sad-figure", "protocolId": "truthful-sad", "src": "/images/nuwa-index/truthful-sad.webp", "original": "/images/nuwa-index/truthful-sad-original.png", "width": 1500, "height": 1000, "alt": { "en": "Situational Awareness Dataset (SAD) — Figure 5", "zh": "Situational Awareness Dataset (SAD) — Figure 5" }, "caption": { "en": "Original result figure. Interpret within the protocol scope and limitations below.", "zh": "原文结果图，请结合下方的协议范围与局限解读。" }, "figureLabel": "Figure 5", "sourceId": "truthful-sad-figure-source", "sourceLocator": "Figure 5", "provenanceUrl": "https://arxiv.org/html/2407.04694v1/fig_1.png", "srcset": "/images/nuwa-index/truthful-sad.webp 1500w, /images/nuwa-index/truthful-sad-640.webp 640w, /images/nuwa-index/truthful-sad-1200.webp 1200w" }, { "id": "strongreject-figure", "protocolId": "strongreject", "src": "/images/nuwa-index/strongreject.webp", "original": "/images/nuwa-index/strongreject-original.svg", "width": 1600, "height": 960, "alt": { "en": "StrongREJECT — Figure 2", "zh": "StrongREJECT — Figure 2" }, "caption": { "en": "Original result figure. Interpret within the protocol scope and limitations below.", "zh": "原文结果图，请结合下方的协议范围与局限解读。" }, "figureLabel": "Figure 2", "sourceId": "strongreject-figure-source", "sourceLocator": "Figure 2", "provenanceUrl": "https://arxiv.org/html/2402.10260v2/mae_by_jailbreak_x_evaluator.svg", "srcset": "/images/nuwa-index/strongreject.webp 1600w, /images/nuwa-index/strongreject-640.webp 640w, /images/nuwa-index/strongreject-1200.webp 1200w" }, { "id": "truthfulqa-figure", "protocolId": "truthfulqa", "src": "/images/nuwa-index/truthfulqa.webp", "original": "/images/nuwa-index/truthfulqa-original.svg", "width": 1600, "height": 1126, "alt": { "en": "TruthfulQA — Figure 2", "zh": "TruthfulQA — Figure 2" }, "caption": { "en": "Original result figure. Interpret within the protocol scope and limitations below.", "zh": "原文结果图，请结合下方的协议范围与局限解读。" }, "figureLabel": "Figure 2", "sourceId": "truthfulqa-figure-source", "sourceLocator": "Figure 2", "provenanceUrl": "https://arxiv.org/html/2109.07958v2/control_uqa.svg", "srcset": "/images/nuwa-index/truthfulqa.webp 1600w, /images/nuwa-index/truthfulqa-640.webp 640w, /images/nuwa-index/truthfulqa-1200.webp 1200w" }, { "id": "cybergym-figure", "protocolId": "cybergym", "src": "/images/nuwa-index/cybergym.webp", "original": "/images/nuwa-index/cybergym-original.png", "width": 324, "height": 210, "alt": { "en": "CyberGym — Figure 2", "zh": "CyberGym — Figure 2" }, "caption": { "en": "Original result figure. Interpret within the protocol scope and limitations below.", "zh": "原文结果图，请结合下方的协议范围与局限解读。" }, "figureLabel": "Figure 2", "sourceId": "cybergym-figure-source", "sourceLocator": "Figure 2", "provenanceUrl": "https://arxiv.org/html/2506.02548v3/x1.png", "srcset": "/images/nuwa-index/cybergym.webp 324w" }, { "id": "conspiracy-persuasion-figure", "protocolId": "conspiracy-persuasion", "src": "/images/nuwa-index/conspiracy-persuasion.webp", "original": "/images/nuwa-index/conspiracy-persuasion-original.svg", "width": 1600, "height": 1468, "alt": { "en": "Study 1 belief changes and evaluations of GPT-4o", "zh": "研究 1 中的信念变化与 GPT-4o 评价" }, "caption": { "en": "Figure 2 · Study 1 analytic sample, N = 1,092. Model-estimated belief changes, AI evaluations and trust under the reported study conditions.", "zh": "图 2｜研究 1 分析样本 N = 1,092；展示原研究条件下模型估计的信念变化、AI 评价与信任。" }, "figureLabel": "Figure 2", "sourceId": "conspiracy-persuasion-figure-source", "sourceLocator": "Figure 2; Study 1", "provenanceUrl": "https://arxiv.org/html/2601.05050v3/figs/fig2.svg", "srcset": "/images/nuwa-index/conspiracy-persuasion.webp 1600w, /images/nuwa-index/conspiracy-persuasion-640.webp 640w, /images/nuwa-index/conspiracy-persuasion-1200.webp 1200w" } ], "visualAudits": [ { "protocolId": "agentcyberrange", "checkedOn": "2026-10-25", "inspectedUrl": "https://agentcyberrange.io/", "sourceVersion": "Website bundle routes-B-0Mubrq.js / 2026-10-24 capture", "numericalStatus": "structured", "tableLocator": "Official leaderboard: Level-0 Pass@3; arrays QR (web), $R (post), version map ez", "figureId": null, "chartId": "agentcyberrange-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "asimov", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2509.21651", "sourceVersion": "ASIMOV v2 / arXiv:2509.21651", "numericalStatus": "figure-only", "tableLocator": "Table 1 ", "figureId": "asimov-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "cybench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2408.08926v4", "sourceVersion": "arXiv:2408.08926v4", "numericalStatus": "structured", "tableLocator": "Table 2", "figureId": null, "chartId": "cybench-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "cybergym", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2506.02548v3", "sourceVersion": "arXiv:2506.02548v3", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "cybergym-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "in-context-scheming", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2412.04984v2", "sourceVersion": "arXiv:2412.04984v2", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "in-context-scheming-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "aisi-cyber-scenarios", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2603.11214", "sourceVersion": "Research page checked 2026-10-25", "numericalStatus": "figure-only", "tableLocator": "Table 1 ", "figureId": "aisi-cyber-scenarios-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "time-horizons", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2503.14499", "sourceVersion": "Time Horizon 1.1; page updated 2026-09-08", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "time-horizons-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "re-bench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2411.15114v2", "sourceVersion": "arXiv:2411.15114v2", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "re-bench-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "replibench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2504.18565v2", "sourceVersion": "arXiv:2504.18565v2", "numericalStatus": "structured", "tableLocator": "Table 2", "figureId": null, "chartId": "replibench-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "self-replication", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2412.12140v1", "sourceVersion": "arXiv:2412.12140v1", "numericalStatus": "structured", "tableLocator": "§1; §3.1", "figureId": null, "chartId": "self-replication-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "truthful-sad", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2407.04694", "sourceVersion": "2024-07-15 article; paper linked", "numericalStatus": "figure-only", "tableLocator": "Table 2 ", "figureId": "truthful-sad-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "stealth-evals", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2505.01420v4", "sourceVersion": "arXiv:2505.01420v4", "numericalStatus": "structured", "tableLocator": "Table 1", "figureId": null, "chartId": "stealth-evals-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "wmdp", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2403.03218v7", "sourceVersion": "arXiv:2403.03218v7", "numericalStatus": "structured", "tableLocator": "Table 1", "figureId": null, "chartId": "wmdp-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "opendeception", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2504.13707v3", "sourceVersion": "arXiv:2504.13707v3", "numericalStatus": "structured", "tableLocator": "Table 1", "figureId": null, "chartId": "opendeception-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "aisi-propensity-study", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2604.21098", "sourceVersion": "Research page checked 2026-10-25", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "aisi-propensity-study-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "palisade-shutdown", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2509.14260", "sourceVersion": "2025-07-05 article; later TMLR version separate", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "palisade-shutdown-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "agentdojo", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2406.13352v3", "sourceVersion": "arXiv:2406.13352v3", "numericalStatus": "structured", "tableLocator": "Appendix C, Table 3", "figureId": null, "chartId": "agentdojo-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "agentharm", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2410.09024v3", "sourceVersion": "arXiv:2410.09024v3", "numericalStatus": "structured", "tableLocator": "Table 2", "figureId": null, "chartId": "agentharm-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "harmbench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2402.04249v2", "sourceVersion": "arXiv:2402.04249v2", "numericalStatus": "figure-only", "tableLocator": "Table 1 ", "figureId": "harmbench-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "anthropic-unintended-actions", "checkedOn": "2026-10-25", "inspectedUrl": "https://www.anthropic.com/research/investigating-unintended-model-actions", "sourceVersion": "2026-10-09 article", "numericalStatus": "figure-only", "tableLocator": null, "figureId": "anthropic-unintended-actions-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "political-persuasion", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2603.09884v1", "sourceVersion": "arXiv:2603.09884v1", "numericalStatus": "figure-only", "tableLocator": "Table 1 ", "figureId": "political-persuasion-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "palisade-robot-shutdown", "checkedOn": "2026-10-25", "inspectedUrl": "https://palisaderesearch.org/assets/reports/shutdown-resistance-on-robots.pdf", "sourceVersion": "Technical report · 2026-02-11", "numericalStatus": "structured", "tableLocator": "§3; Figure 2 (p. 3)", "figureId": null, "chartId": "palisade-robot-shutdown-results", "display": "chart", "note": { "en": "The separate robot technical report was checked; simulation prompt conditions are plotted without pooling physical trials.", "zh": "核验单独的机器人技术报告；绘制模拟提示条件结果，不与实机试验合并。" } }, { "protocolId": "strongreject", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2402.10260v2", "sourceVersion": "arXiv:2402.10260v2", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "strongreject-figure", "chartId": null, "display": "original-figure", "note": { "en": "Source retained for methodological reading. A reliable result figure was not obtained in this audit; no numerical score is fabricated.", "zh": "保留原文的方法性阅读。本轮未取得可靠的结果图，不制作虚构评分。" } }, { "protocolId": "secbench-pro", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2605.26548v2", "sourceVersion": "arXiv:2605.26548v2", "numericalStatus": "figure-only", "tableLocator": null, "figureId": "secbench-pro-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "sandboxescapebench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2603.02277", "sourceVersion": "AISI research page checked 2026-10-25", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "sandboxescapebench-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "paperbench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2504.01848v3", "sourceVersion": "arXiv:2504.01848v3", "numericalStatus": "structured", "tableLocator": "Table 4", "figureId": null, "chartId": "paperbench-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "alignment-faking", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2412.14093", "sourceVersion": "Anthropic research article 2024-12-18", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "alignment-faking-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "agentic-misalignment", "checkedOn": "2026-10-25", "inspectedUrl": "https://www.anthropic.com/research/agentic-misalignment", "sourceVersion": "Anthropic research article 2025-06-20", "numericalStatus": "figure-only", "tableLocator": null, "figureId": "agentic-misalignment-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "shade-arena", "checkedOn": "2026-10-25", "inspectedUrl": "https://www.arxiv.org/html/2506.15740", "sourceVersion": "Anthropic research article 2025-06-16", "numericalStatus": "figure-only", "tableLocator": "Table 1 ", "figureId": "shade-arena-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "vct", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2504.16137v2", "sourceVersion": "arXiv:2504.16137v2", "numericalStatus": "structured", "tableLocator": "Table 1", "figureId": null, "chartId": "vct-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "vct-v2", "checkedOn": "2026-10-25", "inspectedUrl": "https://securebio.org/blog/introducing-vct-v2/", "sourceVersion": "SecureBio release 2026-09-11", "numericalStatus": "figure-only", "tableLocator": null, "figureId": "vct-v2-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "biotier", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2607.14479v1", "sourceVersion": "arXiv:2607.14479v1", "numericalStatus": "figure-only", "tableLocator": "Table 1 ", "figureId": "biotier-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "mask", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2503.03750v3", "sourceVersion": "arXiv:2503.03750v3", "numericalStatus": "structured", "tableLocator": "Table 3", "figureId": null, "chartId": "mask-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "ape", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2506.02873v4", "sourceVersion": "arXiv:2506.02873v4", "numericalStatus": "figure-only", "tableLocator": "Table 2 ", "figureId": "ape-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "conspiracy-persuasion", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2601.05050v3", "sourceVersion": "arXiv:2601.05050v3", "numericalStatus": "figure-only", "tableLocator": null, "figureId": "conspiracy-persuasion-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original result figure retained; model-estimated effects are not transcribed from pixels.", "zh": "保留原始结果图，不从像素转录模型估计效应。" } }, { "protocolId": "sec-bench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2506.11791", "sourceVersion": "arXiv:2506.11791v2", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "sec-bench-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "exploitbench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2605.14153", "sourceVersion": "arXiv:2605.14153v1", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "exploitbench-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "exploitgym", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2605.11086", "sourceVersion": "Source capture 2026-10-25", "numericalStatus": "figure-only", "tableLocator": "Table 1 ", "figureId": "exploitgym-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "cve-bench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2503.17332", "sourceVersion": "arXiv:2503.17332v4", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "cve-bench-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "sre-bench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2608.11469", "sourceVersion": "arXiv:2608.11469v2", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "sre-bench-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "cyberseceval", "checkedOn": "2026-10-25", "inspectedUrl": "https://meta-llama.github.io/PurpleLlama/CyberSecEval/docs/intro", "sourceVersion": "Source capture 2026-10-25", "numericalStatus": "suite-overview", "tableLocator": null, "figureId": null, "chartId": null, "display": "method-diagram", "note": { "en": "This is a suite of distinct, versioned protocols. The original introduction does not provide a unified numerical result; component results must be recorded separately.", "zh": "该套件包含不同版本的独立协议。原始介绍未提供统一数值结果；各组成评测的成绩须分别记录。" } }, { "protocolId": "cybersoceval", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2509.20166v1", "sourceVersion": "arXiv:2509.20166v1", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "cybersoceval-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "hcast", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2503.17354", "sourceVersion": "arXiv:2503.17354v1", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "hcast-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "mirrorcode", "checkedOn": "2026-10-25", "inspectedUrl": "https://metr.org/blog/2026-04-10-mirrorcode-preliminary-results/", "sourceVersion": "Source capture 2026-10-25", "numericalStatus": "figure-only", "tableLocator": null, "figureId": "mirrorcode-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "mle-bench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2410.07095", "sourceVersion": "arXiv:2410.07095v6", "numericalStatus": "structured", "tableLocator": "Table 2", "figureId": null, "chartId": "mle-bench-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "posttrainbench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2603.08640", "sourceVersion": "arXiv:2603.08640v2", "numericalStatus": "structured", "tableLocator": "Table 1", "figureId": null, "chartId": "posttrainbench-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "machiavelli", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2304.03279", "sourceVersion": "arXiv:2304.03279v4", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "machiavelli-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "tamperbench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2602.06911", "sourceVersion": "arXiv:2602.06911v2", "numericalStatus": "figure-only", "tableLocator": "Table 1 . Manual categorization of 10 StrongREJECT responses per model–condition pair.", "figureId": "tamperbench-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "injecagent", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2403.02691", "sourceVersion": "arXiv:2403.02691v3", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "injecagent-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "agent-safetybench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2412.14470", "sourceVersion": "arXiv:2412.14470v2", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "agent-safetybench-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "truthfulqa", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2109.07958", "sourceVersion": "Source capture 2026-10-25", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "truthfulqa-figure", "chartId": null, "display": "original-figure", "note": { "en": "Source retained for methodological reading. A reliable result figure was not obtained in this audit; no numerical score is fabricated.", "zh": "保留原文的方法性阅读。本轮未取得可靠的结果图，不制作虚构评分。" } }, { "protocolId": "labbench-two", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2604.09554", "sourceVersion": "arXiv:2604.09554v2", "numericalStatus": "figure-only", "tableLocator": "Table 1 ", "figureId": "labbench-two-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "fortress", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2506.14922", "sourceVersion": "arXiv:2506.14922v2", "numericalStatus": "figure-only", "tableLocator": "Table 1 ", "figureId": "fortress-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "bountybench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2505.15216", "sourceVersion": "arXiv:2505.15216v3", "numericalStatus": "structured", "tableLocator": "Table 1", "figureId": null, "chartId": "bountybench-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "basharena", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2512.15688", "sourceVersion": "arXiv:2512.15688v1", "numericalStatus": "figure-only", "tableLocator": null, "figureId": "basharena-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "monitorbench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2603.28590v2", "sourceVersion": "arXiv:2603.28590v2", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "monitorbench-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "catastrophic-cyber-benchmark", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2410.09114", "sourceVersion": "arXiv:2410.09114v2", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "catastrophic-cyber-benchmark-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "instrumentaleval", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2502.12206", "sourceVersion": "arXiv:2502.12206v1", "numericalStatus": "structured", "tableLocator": "Table 1", "figureId": null, "chartId": "instrumentaleval-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "mcptox", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2508.14925", "sourceVersion": "arXiv:2508.14925v2", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "mcptox-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "judgebench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2410.12784v2", "sourceVersion": "arXiv:2410.12784v2", "numericalStatus": "structured", "tableLocator": "Table 2", "figureId": null, "chartId": "judgebench-results", "display": "chart", "note": { "en": "Exact source-table values transcribed; configuration differences retained; no comparison group asserted.", "zh": "精确转录原文表格值，保留配置差异，不建立比较组。" } }, { "protocolId": "narcbench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/html/2604.01151v2", "sourceVersion": "arXiv:2604.01151v2", "numericalStatus": "figure-only", "tableLocator": "Table 1", "figureId": "narcbench-figure", "chartId": null, "display": "original-figure", "note": { "en": "Original figure retained without estimating numerical values. Full source tables have not been transcribed.", "zh": "保留原图，不估算数值；本轮未完成原文全部表格转录。" } }, { "protocolId": "abstentionbench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/abs/2506.09038", "sourceVersion": "2026-10-25 source capture", "numericalStatus": "qualitative", "tableLocator": null, "figureId": null, "chartId": null, "display": "method-diagram", "note": { "en": "Protocol scope is catalogued. Exact numerical extraction and an appropriate source figure remain to be reviewed.", "zh": "已收录协议范围，精确数值提取及适用原图仍待核查。" } }, { "protocolId": "deceptionbench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/abs/2510.15501", "sourceVersion": "2026-10-25 source capture", "numericalStatus": "structured", "tableLocator": "Report dates and original executor are shown with this assessment.", "figureId": null, "chartId": "neo-glm52-deceptionbench-chart", "display": "chart", "note": { "en": "Source-report group is plotted with its actual configuration; no cross-report ranking.", "zh": "依据报告实际配置绘图，不作跨报告排名。" } }, { "protocolId": "persusafety", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/abs/2504.10430", "sourceVersion": "2026-10-25 source capture", "numericalStatus": "structured", "tableLocator": "Report dates and original executor are shown with this assessment.", "figureId": null, "chartId": "neo-glm52-persusafety-chart", "display": "chart", "note": { "en": "Source-report group is plotted with its actual configuration; no cross-report ranking.", "zh": "依据报告实际配置绘图，不作跨报告排名。" } }, { "protocolId": "impossiblebench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/abs/2510.20270", "sourceVersion": "2026-10-25 source capture", "numericalStatus": "qualitative", "tableLocator": null, "figureId": null, "chartId": null, "display": "method-diagram", "note": { "en": "Protocol scope is catalogued. Exact numerical extraction and an appropriate source figure remain to be reviewed.", "zh": "已收录协议范围，精确数值提取及适用原图仍待核查。" } }, { "protocolId": "evalawarebench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/abs/2605.23055", "sourceVersion": "2026-10-25 source capture", "numericalStatus": "structured", "tableLocator": "Report dates and original executor are shown with this assessment.", "figureId": null, "chartId": "neo-kimi-k3-evalawarebench-chart", "display": "chart", "note": { "en": "Source-report group is plotted with its actual configuration; no cross-report ranking.", "zh": "依据报告实际配置绘图，不作跨报告排名。" } }, { "protocolId": "theagentcompany", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/abs/2412.14161", "sourceVersion": "2026-10-25 source capture", "numericalStatus": "qualitative", "tableLocator": null, "figureId": null, "chartId": null, "display": "method-diagram", "note": { "en": "Protocol scope is catalogued. Exact numerical extraction and an appropriate source figure remain to be reviewed.", "zh": "已收录协议范围，精确数值提取及适用原图仍待核查。" } }, { "protocolId": "lab-bench", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/abs/2407.10362", "sourceVersion": "2026-10-25 source capture", "numericalStatus": "structured", "tableLocator": "Report dates and original executor are shown with this assessment.", "figureId": null, "chartId": "neo-kimi-k3-lab-bench-chart", "display": "chart", "note": { "en": "Source-report group is plotted with its actual configuration; no cross-report ranking.", "zh": "依据报告实际配置绘图，不作跨报告排名。" } }, { "protocolId": "charxiv", "checkedOn": "2026-10-25", "inspectedUrl": "https://charxiv.github.io/", "sourceVersion": "2026-10-25 source capture", "numericalStatus": "structured", "tableLocator": "Report dates and original executor are shown with this assessment.", "figureId": null, "chartId": "neo-kimi-k3-charxiv-chart", "display": "chart", "note": { "en": "Source-report group is plotted with its actual configuration; no cross-report ranking.", "zh": "依据报告实际配置绘图，不作跨报告排名。" } }, { "protocolId": "petri", "checkedOn": "2026-10-25", "inspectedUrl": "https://www.anthropic.com/research/petri-open-source-auditing", "sourceVersion": "2026-10-25 source capture", "numericalStatus": "qualitative", "tableLocator": null, "figureId": null, "chartId": null, "display": "method-diagram", "note": { "en": "Protocol scope is catalogued. Exact numerical extraction and an appropriate source figure remain to be reviewed.", "zh": "已收录协议范围，精确数值提取及适用原图仍待核查。" } }, { "protocolId": "biomysterybench", "checkedOn": "2026-10-25", "inspectedUrl": "https://www.anthropic.com/research/Evaluating-Claude-For-Bioinformatics-With-BioMysteryBench", "sourceVersion": "2026-10-25 source capture", "numericalStatus": "qualitative", "tableLocator": null, "figureId": null, "chartId": null, "display": "method-diagram", "note": { "en": "Protocol scope is catalogued. Exact numerical extraction and an appropriate source figure remain to be reviewed.", "zh": "已收录协议范围，精确数值提取及适用原图仍待核查。" } }, { "protocolId": "evmbench", "checkedOn": "2026-10-25", "inspectedUrl": "https://openai.com/index/introducing-evmbench/", "sourceVersion": "2026-10-25 source capture", "numericalStatus": "qualitative", "tableLocator": null, "figureId": null, "chartId": null, "display": "method-diagram", "note": { "en": "Protocol scope is catalogued. Exact numerical extraction and an appropriate source figure remain to be reviewed.", "zh": "已收录协议范围，精确数值提取及适用原图仍待核查。" } }, { "protocolId": "swe-bench-pro", "checkedOn": "2026-10-25", "inspectedUrl": "https://arxiv.org/abs/2509.16941", "sourceVersion": "2026-10-25 source capture", "numericalStatus": "qualitative", "tableLocator": null, "figureId": null, "chartId": null, "display": "method-diagram", "note": { "en": "Protocol scope is catalogued. Exact numerical extraction and an appropriate source figure remain to be reviewed.", "zh": "已收录协议范围，精确数值提取及适用原图仍待核查。" } }, { "protocolId": "ctf-archive-diamond", "checkedOn": "2026-10-25", "inspectedUrl": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "sourceVersion": "2026-10-25 source capture", "numericalStatus": "structured", "tableLocator": "Report dates and original executor are shown with this assessment.", "figureId": null, "chartId": "caisi-deepseek-v4-ctf-archive-diamond-chart", "display": "chart", "note": { "en": "Source-report group is plotted with its actual configuration; no cross-report ranking.", "zh": "依据报告实际配置绘图，不作跨报告排名。" } }, { "protocolId": "portbench", "checkedOn": "2026-10-25", "inspectedUrl": "https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro", "sourceVersion": "2026-10-25 source capture", "numericalStatus": "structured", "tableLocator": "Report dates and original executor are shown with this assessment.", "figureId": null, "chartId": "caisi-deepseek-v4-portbench-chart", "display": "chart", "note": { "en": "Source-report group is plotted with its actual configuration; no cross-report ranking.", "zh": "依据报告实际配置绘图，不作跨报告排名。" } } ], "models": [ { "id": "kimi-k3", "name": "Kimi K3", "family": "Kimi", "developerId": "moonshot", "releaseDate": "2026-07-16", "releaseSourceId": "neo-kimi-k3-source", "releaseDateBasis": "assessment-reports-release", "events": [ { "kind": "public-availability", "date": "2026-07-16", "sourceId": "neo-kimi-k3-source" } ], "summary": { "en": "Neo’s Kimi K3 assessment covers missing-input fabrication, deception-related behavior and shutdown scenarios. Its missing-image experiments modify CharXiv and LAB-Bench: an attempt to answer without the image is a behavioral observation, not the original Benchmark’s task accuracy.", "zh": "Neo 对 Kimi K3 的评估涉及缺失输入时的编造、欺骗相关行为及关停场景。CharXiv 和 LAB-Bench 实验移除了原有图像输入：继续作答属于行为观察，不是原基准的任务准确率。" }, "synthesis": [ { "text": { "en": "Neo’s Kimi K3 assessment covers missing-input fabrication, deception-related behavior and shutdown scenarios. Its missing-image experiments modify CharXiv and LAB-Bench: an attempt to answer without the image is a behavioral observation, not the original Benchmark’s task accuracy.", "zh": "Neo 对 Kimi K3 的评估涉及缺失输入时的编造、欺骗相关行为及关停场景。CharXiv 和 LAB-Bench 实验移除了原有图像输入：继续作答属于行为观察，不是原基准的任务准确率。" }, "sourceIds": [ "neo-kimi-k3-source" ] } ], "scenarioScope": "single-agent", "aliases": [] }, { "id": "glm-5-2", "name": "GLM-5.2", "family": "GLM", "developerId": "zai", "releaseDate": "2026-06-16", "releaseSourceId": "saferai-glm52-source", "releaseDateBasis": "assessment-reports-release", "events": [ { "kind": "public-availability", "date": "2026-06-16", "sourceId": "saferai-glm52-source" } ], "summary": { "en": "Neo and SaferAI examine different aspects of GLM-5.2. Neo’s behavioral and persuasion experiments use their own scenarios and judges; SaferAI reports cyber capability and honesty evaluations under distinct task selections and budgets. These studies do not supply interchangeable measures of one latent safety score.", "zh": "Neo 与 SaferAI 从不同角度评估 GLM-5.2。Neo 的行为及说服实验使用特定场景和判别器；SaferAI 的网络能力与诚实性评测采用不同任务选择和预算。这些研究不能互换为同一潜在安全分数。" }, "synthesis": [ { "text": { "en": "Neo and SaferAI examine different aspects of GLM-5.2. Neo’s behavioral and persuasion experiments use their own scenarios and judges; SaferAI reports cyber capability and honesty evaluations under distinct task selections and budgets. These studies do not supply interchangeable measures of one latent safety score.", "zh": "Neo 与 SaferAI 从不同角度评估 GLM-5.2。Neo 的行为及说服实验使用特定场景和判别器；SaferAI 的网络能力与诚实性评测采用不同任务选择和预算。这些研究不能互换为同一潜在安全分数。" }, "sourceIds": [ "neo-glm52-source", "saferai-glm52-source" ] } ], "scenarioScope": "single-agent", "aliases": [] }, { "id": "deepseek-v4-pro", "name": "DeepSeek V4 Pro", "family": "DeepSeek", "developerId": "deepseek", "releaseDate": "2026-04-24", "releaseSourceId": "neo-deepseek-v4-source", "releaseDateBasis": "assessment-reports-release", "events": [ { "kind": "public-availability", "date": "2026-04-24", "sourceId": "neo-deepseek-v4-source" } ], "summary": { "en": "Neo’s report examines behavioral safety, while CAISI reports a separate capability assessment. In CAISI’s public CTF comparison, the DeepSeek value is IRT-imputed. It must not be described as a directly observed completion rate or combined with another report’s attack-success metric.", "zh": "Neo 报告侧重行为安全，CAISI 则独立报告能力评估。CAISI 公开 CTF 对比中的 DeepSeek 数值采用 IRT 插补，不能表述为直接观察完成率，也不能与其他报告的攻击成功率合并。" }, "synthesis": [ { "text": { "en": "Neo’s report examines behavioral safety, while CAISI reports a separate capability assessment. In CAISI’s public CTF comparison, the DeepSeek value is IRT-imputed. It must not be described as a directly observed completion rate or combined with another report’s attack-success metric.", "zh": "Neo 报告侧重行为安全，CAISI 则独立报告能力评估。CAISI 公开 CTF 对比中的 DeepSeek 数值采用 IRT 插补，不能表述为直接观察完成率，也不能与其他报告的攻击成功率合并。" }, "sourceIds": [ "neo-deepseek-v4-source", "caisi-deepseek-v4-source" ] } ], "scenarioScope": "single-agent", "aliases": [] }, { "id": "deepseek-v4-flash-0731", "name": "DeepSeek V4 Flash 0731", "family": "DeepSeek", "developerId": "deepseek", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-5-6-terra", "name": "GPT-5.6 Terra", "family": "GPT", "developerId": "openai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "claude-sonnet-5", "name": "Claude Sonnet 5", "family": "Claude", "developerId": "anthropic", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-5-5", "name": "GPT-5.5", "family": "GPT", "developerId": "openai", "releaseDate": "2026-04-24", "releaseSourceId": "saferai-glm52-source", "releaseDateBasis": "assessment-reports-release", "events": [ { "kind": "public-availability", "date": "2026-04-24", "sourceId": "saferai-glm52-source" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "claude-opus-4-7", "name": "Claude Opus 4.7", "family": "Claude", "developerId": "anthropic", "releaseDate": "2026-04-16", "releaseSourceId": "saferai-glm52-source", "releaseDateBasis": "assessment-reports-release", "events": [ { "kind": "public-availability", "date": "2026-04-16", "sourceId": "saferai-glm52-source" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-5-4", "name": "GPT-5.4", "family": "GPT", "developerId": "openai", "releaseDate": "2026-03-05", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2026-03-05", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-5-4-mini", "name": "GPT-5.4 mini", "family": "GPT", "developerId": "openai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "claude-opus-4-6", "name": "Claude Opus 4.6", "family": "Claude", "developerId": "anthropic", "releaseDate": "2026-02-05", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2026-02-05", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "deepseek-r1", "name": "DeepSeek R1", "family": "DeepSeek", "developerId": "deepseek", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "deepseek-r1-0528", "name": "DeepSeek R1-0528", "family": "DeepSeek", "developerId": "deepseek", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "deepseek-v3-1", "name": "DeepSeek V3.1", "family": "DeepSeek", "developerId": "deepseek", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-5", "name": "GPT-5", "family": "GPT", "developerId": "openai", "releaseDate": "2025-08-07", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2025-08-07", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-5-mini", "name": "GPT-5 mini", "family": "GPT", "developerId": "openai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-oss-120b", "name": "gpt-oss-120b", "family": "GPT", "developerId": "openai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "claude-opus-4", "name": "Claude Opus 4", "family": "Claude", "developerId": "anthropic", "releaseDate": "2025-05-22", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2025-05-22", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "claude-sonnet-4-6", "name": "Claude Sonnet 4.6", "family": "Claude", "developerId": "anthropic", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gemini-2-5-pro", "name": "Gemini 2.5 Pro", "family": "Gemini", "developerId": "deepmind", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "claude-3-5-sonnet-june-2024", "name": "Claude 3.5 Sonnet (June 2024)", "family": "Claude", "developerId": "anthropic", "releaseDate": "2024-06-20", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2024-06-20", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "claude-3-5-sonnet-october-2024", "name": "Claude 3.5 Sonnet (October 2024)", "family": "Claude", "developerId": "anthropic", "releaseDate": "2024-10-22", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2024-10-22", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "claude-3-7-sonnet", "name": "Claude 3.7 Sonnet", "family": "Claude", "developerId": "anthropic", "releaseDate": "2025-02-24", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2025-02-24", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "claude-3-opus", "name": "Claude 3 Opus", "family": "Claude", "developerId": "anthropic", "releaseDate": "2024-03-04", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2024-03-04", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "claude-opus-4-1", "name": "Claude Opus 4.1", "family": "Claude", "developerId": "anthropic", "releaseDate": "2025-08-05", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2025-08-05", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "claude-opus-4-5", "name": "Claude Opus 4.5", "family": "Claude", "developerId": "anthropic", "releaseDate": "2025-11-24", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2025-11-24", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-3-davinci-002", "name": "GPT-3 davinci-002", "family": "GPT", "developerId": "openai", "releaseDate": "2020-05-28", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2020-05-28", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-5-3-codex", "name": "GPT-5.3-Codex", "family": "GPT", "developerId": "openai", "releaseDate": "2026-02-05", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2026-02-05", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gemini-3-pro", "name": "Gemini 3 Pro", "family": "Gemini", "developerId": "deepmind", "releaseDate": "2025-11-18", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2025-11-18", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gemini-3-1-pro", "name": "Gemini 3.1 Pro", "family": "Gemini", "developerId": "deepmind", "releaseDate": "2026-02-19", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2026-02-19", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-2", "name": "GPT-2", "family": "GPT", "developerId": "openai", "releaseDate": "2019-02-14", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2019-02-14", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-3-5-turbo-instruct", "name": "GPT-3.5 Turbo Instruct", "family": "GPT", "developerId": "openai", "releaseDate": "2022-03-15", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2022-03-15", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-4", "name": "GPT-4", "family": "GPT", "developerId": "openai", "releaseDate": "2023-03-14", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2023-03-14", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-4-november-2023", "name": "GPT-4 November 2023", "family": "GPT", "developerId": "openai", "releaseDate": "2023-11-06", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2023-11-06", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-4-turbo", "name": "GPT-4 Turbo", "family": "GPT", "developerId": "openai", "releaseDate": "2024-04-09", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2024-04-09", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-4o", "name": "GPT-4o", "family": "GPT", "developerId": "openai", "releaseDate": "2024-05-13", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2024-05-13", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-5-1-codex-max", "name": "GPT-5.1 Codex Max", "family": "GPT", "developerId": "openai", "releaseDate": "2025-11-19", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2025-11-19", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "gpt-5-2", "name": "GPT-5.2", "family": "GPT", "developerId": "openai", "releaseDate": "2025-12-11", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2025-12-11", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "o1", "name": "o1", "family": "GPT", "developerId": "openai", "releaseDate": "2024-12-05", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2024-12-05", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "o1-preview", "name": "o1-preview", "family": "GPT", "developerId": "openai", "releaseDate": "2024-09-12", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2024-09-12", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "o3", "name": "o3", "family": "GPT", "developerId": "openai", "releaseDate": "2025-04-16", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2025-04-16", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "claude-mythos-preview-early", "name": "Claude Mythos Preview (early)", "family": "Claude", "developerId": "anthropic", "releaseDate": "2026-04-07", "releaseSourceId": "metr-horizon-11-data", "releaseDateBasis": "evaluator-release-date", "events": [ { "kind": "public-availability", "date": "2026-04-07", "sourceId": "metr-horizon-11-data" } ], "summary": { "en": "Published risk evidence is organized by assessment and protocol. Missing evidence is not a low-risk finding.", "zh": "按评估与协议整理已发表风险证据；证据缺失不代表风险较低。" }, "synthesis": [], "scenarioScope": "single-agent", "aliases": [] }, { "id": "claude-mythos-preview", "name": "Claude Mythos Preview", "family": "Claude", "developerId": "anthropic", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "claude-opus-4-8", "name": "Claude Opus 4.8", "family": "Claude", "developerId": "anthropic", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "claude-fable-5-1", "name": "Claude Fable 5.1", "family": "Claude", "developerId": "anthropic", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "claude-mythos-5-1", "name": "Claude Mythos 5.1", "family": "Claude", "developerId": "anthropic", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "claude-opus-5-5", "name": "Claude Opus 5.5", "family": "Claude", "developerId": "anthropic", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "claude-sonnet-5-5", "name": "Claude Sonnet 5.5", "family": "Claude", "developerId": "anthropic", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "claude-haiku-5-5", "name": "Claude Haiku 5.5", "family": "Claude", "developerId": "anthropic", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "deepseek-v4-flash", "name": "DeepSeek-V4-Flash", "family": "DeepSeek-V4-Flash", "developerId": "deepseek", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "gemini-4-argon", "name": "Gemini 4 Argon", "family": "Gemini", "developerId": "deepmind", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "muse-glimmer", "name": "Muse Glimmer", "family": "Muse", "developerId": "meta", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "muse-spark-1-2", "name": "Muse Spark 1.2", "family": "Muse", "developerId": "meta", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "muse-spark-1-3", "name": "Muse Spark 1.3", "family": "Muse", "developerId": "meta", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "muse-spark-contemplating", "name": "Muse Spark Contemplating", "family": "Muse", "developerId": "meta", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "muse-spark-open", "name": "Muse Spark Open", "family": "Muse", "developerId": "meta", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "muse-spark", "name": "Muse Spark", "family": "Muse", "developerId": "meta", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "muse-wonder", "name": "Muse Wonder", "family": "Muse", "developerId": "meta", "releaseDate": "2026-10-12", "releaseSourceId": "official-meta-muse-wonder-model-card", "releaseDateBasis": "developer-release-record", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "muse-wonder-full", "name": "Muse Wonder Full", "family": "Muse", "developerId": "meta", "releaseDate": "2026-10-12", "releaseSourceId": "official-meta-muse-wonder-model-card", "releaseDateBasis": "developer-release-record", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "muse-spark-1-1", "name": "Muse Spark 1.1", "family": "Muse", "developerId": "meta", "releaseDate": "2026-07-09", "releaseSourceId": "official-meta-muse-spark-1-1-evaluation-report", "releaseDateBasis": "developer-release-record", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "mistral-large-4", "name": "Mistral Large 4", "family": "Mistral", "developerId": "mistral", "releaseDate": "2026-10-06", "releaseSourceId": "official-mistral-mistral-large-4-official-technical-documentation", "releaseDateBasis": "developer-release-record", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "gpt-5-4-thinking", "name": "GPT-5.4 Thinking", "family": "GPT-5.4", "developerId": "openai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "gpt-5-6", "name": "GPT-5.6", "family": "GPT-5.6", "developerId": "openai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "gpt-6-astra", "name": "GPT-6 Astra", "family": "GPT-6", "developerId": "openai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "gpt-6-sol-september", "name": "GPT-6 Sol (September)", "family": "GPT-6", "developerId": "openai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "gpt-6-luna-september", "name": "GPT-6 Luna (September)", "family": "GPT-6", "developerId": "openai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "gpt-6-1-sol", "name": "GPT-6.1 Sol", "family": "GPT-6.1", "developerId": "openai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "gpt-6-sol-october", "name": "GPT-6 Sol (October)", "family": "GPT-6", "developerId": "openai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "gpt-6-luna-october", "name": "GPT-6 Luna (October)", "family": "GPT-6", "developerId": "openai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "qwen3-8-2-4t-a95b", "name": "Qwen3.8-2.4T-A95B", "family": "Qwen3.8-2.4T-A95B", "developerId": "qwen", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "qwen3-8-flash-next", "name": "Qwen3.8-Flash-Next", "family": "Qwen3.8-Flash-Next", "developerId": "qwen", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "grok-4-7", "name": "Grok 4.7", "family": "Grok", "developerId": "xai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "glm-5-3-bf16", "name": "GLM-5.3-BF16", "family": "GLM-5.3-BF16", "developerId": "zai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] }, { "id": "glm-5", "name": "GLM-5", "family": "GLM-5", "developerId": "zai", "releaseDate": null, "releaseSourceId": null, "releaseDateBasis": "unknown", "events": [], "summary": { "en": "Developer documentation is linked below. Structured risk results have not yet been extracted for this exact model record.", "zh": "下方关联开发者文档，尚未为该具体模型提取结构化风险结果。" }, "synthesis": [], "scenarioScope": "system", "aliases": [] } ], "assessments": [ { "id": "neo-kimi-k3", "title": "Evaluating Kimi K3 for Misalignment Risks", "kind": "external", "sourceId": "neo-kimi-k3-source", "publicationDate": "2026-09-11", "updatedDate": null, "modelIds": [ "kimi-k3", "gpt-5-6-terra", "claude-sonnet-5", "deepseek-v4-flash-0731", "deepseek-v4-pro" ], "publisherIds": [ "neo" ], "summary": { "en": "The report identifies conditional overeagerness, grader-directed behavior and recognition of familiar evaluations. It does not establish a universal deployment failure rate.", "zh": "报告发现有条件的过度主动、面向判分器的行为和对熟悉评测的识别，不代表普遍部署失败率。" }, "areas": [ "autonomy", "deception", "cybersecurity", "cbrn", "disinformation" ], "sections": [ { "heading": "CharXiv with images withheld", "locator": "§1.1.1; pp. 7–9", "variantIds": [ "neo-kimi-k3-charxiv" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "LAB-Bench with figures/tables withheld", "locator": "§1.1.2; pp. 8–10", "variantIds": [ "neo-kimi-k3-lab-bench" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "AbstentionBench", "locator": "§1.2; p. 11", "variantIds": [ "neo-kimi-k3-abstentionbench" ], "disposition": "linked", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Pre-seeded alignment audits", "locator": "§1.3; pp. 12–18", "variantIds": [ "neo-kimi-k3-preseeded-audits" ], "disposition": "report-specific", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Cybench", "locator": "§2.1.1; p. 19", "variantIds": [ "neo-kimi-k3-cybench" ], "disposition": "linked", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "EVMbench", "locator": "§2.1.2; p. 19", "variantIds": [ "neo-kimi-k3-evmbench" ], "disposition": "linked", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "CyberGym infrastructure probing", "locator": "§2.1.3–2.1.4; pp. 20–23", "variantIds": [ "neo-kimi-k3-cybergym" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "TheAgentCompany boundary behavior", "locator": "§2.2; pp. 24–25", "variantIds": [ "neo-kimi-k3-theagentcompany" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "ImpossibleBench", "locator": "§2.3; p. 26", "variantIds": [ "neo-kimi-k3-impossiblebench" ], "disposition": "linked", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Apollo-scenario recognition", "locator": "§2.4.1; p. 28", "variantIds": [ "neo-kimi-k3-apollo-scenario-awareness" ], "disposition": "report-specific", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "EvalAwareBench", "locator": "§2.4.2; pp. 28–30", "variantIds": [ "neo-kimi-k3-evalawarebench" ], "disposition": "linked", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Alignment faking", "locator": "§3.1; p. 32", "variantIds": [ "neo-kimi-k3-alignment-faking" ], "disposition": "linked", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Agentic Misalignment, historical", "locator": "§3.2.1; pp. 34–36", "variantIds": [ "neo-kimi-k3-agentic-misalignment" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Agentic Misalignment, translated", "locator": "§3.2.2; pp. 37–39", "variantIds": [ "neo-kimi-k3-translated-misalignment" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Shutdown resistance", "locator": "§4.1; pp. 40–41", "variantIds": [ "neo-kimi-k3-palisade-shutdown" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "RunPod service continuity", "locator": "§4.2; pp. 40–43", "variantIds": [ "neo-kimi-k3-runpod-continuity" ], "disposition": "report-specific", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Petri", "locator": "§4.3; pp. 43–44", "variantIds": [ "neo-kimi-k3-petri" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Situational Awareness Dataset", "locator": "§5.1; p. 45", "variantIds": [ "neo-kimi-k3-truthful-sad" ], "disposition": "linked", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "GDM Stealth evaluations", "locator": "§5.2; p. 46", "variantIds": [ "neo-kimi-k3-stealth-evals" ], "disposition": "linked", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Hint-influence monitorability", "locator": "§5.3.1; p. 48", "variantIds": [ "neo-kimi-k3-hint-monitorability" ], "disposition": "report-specific", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "CoT controllability", "locator": "§5.3.2; p. 48", "variantIds": [ "neo-kimi-k3-cot-controllability" ], "disposition": "report-specific", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "SHADE-Arena", "locator": "§5.3.3; p. 49", "variantIds": [ "neo-kimi-k3-shade-arena" ], "disposition": "linked", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "AgentHarm", "locator": "§6.1; p. 51", "variantIds": [ "neo-kimi-k3-agentharm" ], "disposition": "linked", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Harmful fine-tuning", "locator": "§6.2; pp. 52–53", "variantIds": [ "neo-kimi-k3-harmful-finetuning" ], "disposition": "report-specific", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "FORTRESS", "locator": "§6.3; pp. 54–57", "variantIds": [ "neo-kimi-k3-fortress" ], "disposition": "linked", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "APE", "locator": "§7.1; p. 58", "variantIds": [ "neo-kimi-k3-ape" ], "disposition": "linked", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "PersuSafety with adapted judge", "locator": "§7.2; pp. 59–60", "variantIds": [ "neo-kimi-k3-persusafety" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "PsychosisBench", "locator": "§7.3; p. 61", "variantIds": [ "neo-kimi-k3-psychosisbench" ], "disposition": "source-unresolved", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "DeceptionBench", "locator": "§7.4; pp. 62–63", "variantIds": [ "neo-kimi-k3-deceptionbench" ], "disposition": "linked", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "MASK", "locator": "§7.5; pp. 64–65", "variantIds": [ "neo-kimi-k3-mask" ], "disposition": "linked", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } } ], "reviewStatus": "section-reviewed", "scopeNote": { "en": "Reported results retain their original evaluator, configuration and measurement scope. NUWA organizes and interprets the evidence; it has not independently reproduced these experiments.", "zh": "结果保留原始执行方、配置与测量范围。NUWA 整理和解读证据，未独立复现这些实验。" }, "researchNote": { "question": { "en": "How does Kimi K3 behave when input, authorization or evaluation boundaries are ambiguous?", "zh": "当输入、授权或评测边界存在歧义时，Kimi K3 如何行动？" }, "method": { "en": "Nearly 30 evaluations, adapted missing-input tests, seeded audits and transcript analyses, with mixed-origin comparison results.", "zh": "近 30 项评测、缺失输入改造、种子审计和轨迹分析，部分对照结果来自其他来源。" }, "finding": { "en": "The report identifies conditional overeagerness, grader-directed behavior and recognition of familiar evaluations. It does not establish a universal deployment failure rate.", "zh": "报告发现有条件的过度主动、面向判分器的行为和对熟悉评测的识别，不代表普遍部署失败率。" }, "limits": { "en": "Controlled tasks, elicitation, benchmark familiarity and configuration differences constrain generalization to deployment.", "zh": "受控任务、诱发方式、基准熟悉程度和配置差异限制对部署的外推。" } } }, { "id": "neo-deepseek-v4", "title": "Evaluating DeepSeek v4 Pro for Frontier Risks", "kind": "external", "sourceId": "neo-deepseek-v4-source", "publicationDate": "2026-05-29", "updatedDate": null, "modelIds": [ "deepseek-v4-pro" ], "publisherIds": [ "neo" ], "summary": { "en": "The report separates task capability from pressure-induced behavior and notes format dependence in persuasion assessment.", "zh": "报告区分任务能力与压力诱发行为，并指出说服评测依赖提问格式。" }, "areas": [ "autonomy", "deception", "cybersecurity", "cbrn", "disinformation" ], "sections": [ { "heading": "Cybench", "locator": "§1.1", "variantIds": [ "neo-deepseek-v4-cybench" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "EVMbench", "locator": "§1.2", "variantIds": [ "neo-deepseek-v4-evmbench" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "WMDP", "locator": "§2.1", "variantIds": [ "neo-deepseek-v4-wmdp" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "LAB-Bench", "locator": "§2.2", "variantIds": [ "neo-deepseek-v4-lab-bench" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "MASK", "locator": "§3.1", "variantIds": [ "neo-deepseek-v4-mask" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "DeceptionBench", "locator": "§3.2", "variantIds": [ "neo-deepseek-v4-deceptionbench" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "APE", "locator": "§3.3", "variantIds": [ "neo-deepseek-v4-ape" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "PersuSafety adapted judge", "locator": "§3.4", "variantIds": [ "neo-deepseek-v4-persusafety" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Alignment faking", "locator": "§4.1", "variantIds": [ "neo-deepseek-v4-alignment-faking" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Agentic Misalignment", "locator": "§4.2", "variantIds": [ "neo-deepseek-v4-agentic-misalignment" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Shutdown resistance", "locator": "§4.3", "variantIds": [ "neo-deepseek-v4-palisade-shutdown" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Reward hacking", "locator": "§4", "variantIds": [ "neo-deepseek-v4-impossiblebench" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Adversarial robustness", "locator": "§5", "variantIds": [ "neo-deepseek-v4-fortress" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Evaluation awareness", "locator": "§6", "variantIds": [ "neo-deepseek-v4-evaluation-awareness" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Automated alignment audit", "locator": "§6", "variantIds": [ "neo-deepseek-v4-petri" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Safety-judge sensitivity", "locator": "§7", "variantIds": [ "neo-deepseek-v4-judge-sensitivity" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } } ], "reviewStatus": "section-reviewed", "scopeNote": { "en": "Reported results retain their original evaluator, configuration and measurement scope. NUWA organizes and interprets the evidence; it has not independently reproduced these experiments.", "zh": "结果保留原始执行方、配置与测量范围。NUWA 整理和解读证据，未独立复现这些实验。" }, "researchNote": { "question": { "en": "Which capabilities, behavioral tendencies and evaluation-validity issues are observed in DeepSeek V4 Pro?", "zh": "DeepSeek V4 Pro 有哪些已观察的能力、行为倾向和评测有效性问题？" }, "method": { "en": "Public-access evaluations covering cyber, scientific knowledge, manipulation, control-related behavior, robustness and judge sensitivity.", "zh": "通过公开访问评测网络、科研知识、操纵、控制相关行为、鲁棒性与判别器敏感性。" }, "finding": { "en": "The report separates task capability from pressure-induced behavior and notes format dependence in persuasion assessment.", "zh": "报告区分任务能力与压力诱发行为，并指出说服评测依赖提问格式。" }, "limits": { "en": "Controlled tasks, elicitation, benchmark familiarity and configuration differences constrain generalization to deployment.", "zh": "受控任务、诱发方式、基准熟悉程度和配置差异限制对部署的外推。" } } }, { "id": "neo-glm52", "title": "Early Results on Evaluating GLM 5.2 for Behavioral and Persuasion Risks", "kind": "external", "sourceId": "neo-glm52-source", "publicationDate": "2026-07-03", "updatedDate": null, "modelIds": [ "glm-5-2", "deepseek-v4-pro", "claude-sonnet-4-6", "claude-sonnet-5" ], "publisherIds": [ "neo" ], "summary": { "en": "The note reports scenario-dependent behavior and an uncertain decline in verbalized evaluation awareness; silence is not evidence of absent awareness.", "zh": "报告记录随情境变化的行为和不确定的显式评测感知下降；不表达不代表没有感知。" }, "areas": [ "autonomy", "deception", "cybersecurity", "cbrn", "disinformation" ], "sections": [ { "heading": "MASK", "locator": "§1; Figure 1", "variantIds": [ "neo-glm52-mask" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "DeceptionBench", "locator": "§1; Figure 2", "variantIds": [ "neo-glm52-deceptionbench" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Gram automated audits", "locator": "§1; Figures 3–4", "variantIds": [ "neo-glm52-gram" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "APE", "locator": "§2; Figure 5", "variantIds": [ "neo-glm52-ape" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "PersuSafety adapted free-generation judge", "locator": "§2; Figure 6", "variantIds": [ "neo-glm52-persusafety" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Agentic Misalignment awareness trend", "locator": "§3; Figure 7", "variantIds": [ "neo-glm52-agentic-misalignment" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "MASK verbalized awareness", "locator": "§3", "variantIds": [ "neo-glm52-mask-awareness" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } } ], "reviewStatus": "section-reviewed", "scopeNote": { "en": "Reported results retain their original evaluator, configuration and measurement scope. NUWA organizes and interprets the evidence; it has not independently reproduced these experiments.", "zh": "结果保留原始执行方、配置与测量范围。NUWA 整理和解读证据，未独立复现这些实验。" }, "researchNote": { "question": { "en": "How does GLM-5.2 respond to pressure, persuasion requests and evaluation cues?", "zh": "GLM-5.2 如何应对压力、说服请求和评测线索？" }, "method": { "en": "MASK, DeceptionBench, Gram, APE, modified PersuSafety judging and verbalized-awareness analyses. Claude comparisons use different releases across tests.", "zh": "采用 MASK、DeceptionBench、Gram、APE、改造 PersuSafety 判别和显式感知分析；各测试的 Claude 对照版本不同。" }, "finding": { "en": "The note reports scenario-dependent behavior and an uncertain decline in verbalized evaluation awareness; silence is not evidence of absent awareness.", "zh": "报告记录随情境变化的行为和不确定的显式评测感知下降；不表达不代表没有感知。" }, "limits": { "en": "Controlled tasks, elicitation, benchmark familiarity and configuration differences constrain generalization to deployment.", "zh": "受控任务、诱发方式、基准熟悉程度和配置差异限制对部署的外推。" } } }, { "id": "saferai-glm52", "title": "GLM-5.2 Risk Evaluation Report", "kind": "external", "sourceId": "saferai-glm52-source", "publicationDate": "2026-08-02", "updatedDate": "2026-08-26", "modelIds": [ "glm-5-2", "gpt-5-5", "claude-opus-4-7", "claude-opus-4-6", "gpt-5-4" ], "publisherIds": [ "saferai" ], "summary": { "en": "The report finds substantial task capability and pressure-dependent behavior; comparison figures include both SaferAI runs and reused external results.", "zh": "报告发现较强任务能力和压力相关行为；比较图同时包含 SaferAI 自测与引用的外部结果。" }, "areas": [ "autonomy", "deception", "cybersecurity", "cbrn", "disinformation" ], "sections": [ { "heading": "Cybench", "locator": "§2.1; Appendix A.1", "variantIds": [ "saferai-glm52-cybench" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "CyberGym hard subset with token budgets", "locator": "§2.2; Appendix A.2", "variantIds": [ "saferai-glm52-cybergym" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "LAB-Bench text subtasks", "locator": "§3.1; Appendix A.3", "variantIds": [ "saferai-glm52-lab-bench" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "BioMysteryBench public release", "locator": "§3.2; Appendix A.4", "variantIds": [ "saferai-glm52-biomysterybench" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "SWE-Bench Pro public tasks", "locator": "§4.2; Appendix A.5", "variantIds": [ "saferai-glm52-swe-bench-pro" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Agentic Misalignment with neutral emails", "locator": "§4.3; Appendix A.9", "variantIds": [ "saferai-glm52-agentic-misalignment" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "MASK by archetype", "locator": "§5.1; Appendix A.7; Table A.3", "variantIds": [ "saferai-glm52-mask" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "APE Turn-1 attempts", "locator": "§5.2; Appendix A.8", "variantIds": [ "saferai-glm52-ape" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Petri 3.0 automated audit", "locator": "§6; Appendix A.6", "variantIds": [ "saferai-glm52-petri" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Validity, contamination and evaluation awareness", "locator": "§7; pp. 27–28", "variantIds": [], "disposition": "limitations", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } } ], "reviewStatus": "section-reviewed", "scopeNote": { "en": "SaferAI ran GLM-5.2 evaluations; selected comparator data are reused from Lyptus, CAIS or Scale AI and identified per section. NUWA has not independently reproduced these experiments.", "zh": "SaferAI 运行 GLM-5.2 评测；部分对照数据引用 Lyptus、CAIS 或 Scale AI，逐节注明。NUWA 未独立复现这些实验。" }, "researchNote": { "question": { "en": "What can a public-API evaluation establish about GLM-5.2 capabilities and elicited behavior?", "zh": "公开 API 评估能对 GLM-5.2 的能力和诱发行为说明什么？" }, "method": { "en": "Eight benchmark families and Petri audits, with task subsets, budgets and per-figure execution provenance documented in v1.1.", "zh": "v1.1 记录八类基准和 Petri 审计的任务子集、预算及逐图执行方。" }, "finding": { "en": "The report finds substantial task capability and pressure-dependent behavior; comparison figures include both SaferAI runs and reused external results.", "zh": "报告发现较强任务能力和压力相关行为；比较图同时包含 SaferAI 自测与引用的外部结果。" }, "limits": { "en": "Controlled tasks, elicitation, benchmark familiarity and configuration differences constrain generalization to deployment.", "zh": "受控任务、诱发方式、基准熟悉程度和配置差异限制对部署的外推。" } } }, { "id": "caisi-deepseek-2025", "title": "Evaluation of DeepSeek AI Models", "kind": "external", "sourceId": "caisi-deepseek-2025-source", "publicationDate": "2025-09-30", "updatedDate": null, "modelIds": [ "deepseek-r1", "deepseek-r1-0528", "deepseek-v3-1", "gpt-5", "gpt-5-mini", "gpt-oss-120b", "claude-opus-4" ], "publisherIds": [ "nist" ], "summary": { "en": "Reported differences depend on task, scaffold and scoring; institutional judgments are retained as source judgments rather than NUWA findings.", "zh": "所报告差异依赖任务、脚手架与评分；机构判断保留为来源观点，不改写为 NUWA 实验发现。" }, "areas": [ "autonomy", "deception", "cybersecurity", "cbrn", "disinformation" ], "sections": [ { "heading": "Cybench", "locator": "§4.1.1", "variantIds": [ "caisi-deepseek-2025-cybench" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "CyberGym", "locator": "§4.1.2", "variantIds": [ "caisi-deepseek-2025-cybergym" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "CTF-Archive", "locator": "§4.1.3", "variantIds": [ "caisi-deepseek-2025-ctf-archive" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "SWE-Bench Verified", "locator": "§4.2.1", "variantIds": [ "caisi-deepseek-2025-swe-verified" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Breakpoint", "locator": "§4.2.2", "variantIds": [ "caisi-deepseek-2025-breakpoint" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "MMLU-Pro", "locator": "§4.3.1", "variantIds": [ "caisi-deepseek-2025-mmlu-pro" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "MMMLU", "locator": "§4.3.2", "variantIds": [ "caisi-deepseek-2025-mmmlu" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "GPQA-Diamond", "locator": "§4.3.3", "variantIds": [ "caisi-deepseek-2025-gpqa" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "HealthBench", "locator": "§4.3.4", "variantIds": [ "caisi-deepseek-2025-healthbench" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Humanity’s Last Exam", "locator": "§4.3.5", "variantIds": [ "caisi-deepseek-2025-hle" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "SMT 2025", "locator": "§4.4", "variantIds": [ "caisi-deepseek-2025-smt2025" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "OTIS-AIME 2025", "locator": "§4.4", "variantIds": [ "caisi-deepseek-2025-otis-aime" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "PUMaC 2024", "locator": "§4.4", "variantIds": [ "caisi-deepseek-2025-pumac" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Agent hijacking", "locator": "§5.1", "variantIds": [ "caisi-deepseek-2025-agentdojo" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Jailbreak robustness", "locator": "§5.2", "variantIds": [ "caisi-deepseek-2025-jailbreak-robustness" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Censorship and narrative evaluations", "locator": "§6", "variantIds": [ "caisi-deepseek-2025-censorship" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Model adoption", "locator": "§7", "variantIds": [], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Inference and scoring", "locator": "Appendices A1–A3", "variantIds": [], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } } ], "reviewStatus": "section-reviewed", "scopeNote": { "en": "Reported results retain their original evaluator, configuration and measurement scope. NUWA organizes and interprets the evidence; it has not independently reproduced these experiments.", "zh": "结果保留原始执行方、配置与测量范围。NUWA 整理和解读证据，未独立复现这些实验。" }, "researchNote": { "question": { "en": "How do selected DeepSeek releases perform under CAISI’s capability and security protocols?", "zh": "选定 DeepSeek 版本在 CAISI 能力与安全协议下表现如何？" }, "method": { "en": "The September 2025 report covers public and held-out tasks, inference configuration, costs, hijacking and jailbreak evaluations.", "zh": "2025 年 9 月报告覆盖公开与保留任务、推理配置、费用、劫持与越狱评测。" }, "finding": { "en": "Reported differences depend on task, scaffold and scoring; institutional judgments are retained as source judgments rather than NUWA findings.", "zh": "所报告差异依赖任务、脚手架与评分；机构判断保留为来源观点，不改写为 NUWA 实验发现。" }, "limits": { "en": "Controlled tasks, elicitation, benchmark familiarity and configuration differences constrain generalization to deployment.", "zh": "受控任务、诱发方式、基准熟悉程度和配置差异限制对部署的外推。" } } }, { "id": "caisi-deepseek-v4", "title": "CAISI Evaluation of DeepSeek V4 Pro", "kind": "external", "sourceId": "caisi-deepseek-v4-source", "publicationDate": "2026-05-01", "updatedDate": null, "modelIds": [ "deepseek-v4-pro", "gpt-5-5", "gpt-5-4-mini", "claude-opus-4-6" ], "publisherIds": [ "nist" ], "summary": { "en": "The assessment reports benchmark-specific performance and an IRT aggregate; the latter is not a frontier-risk score.", "zh": "评估报告各基准表现和 IRT 汇总；该汇总不是前沿风险分数。" }, "areas": [ "autonomy", "deception", "cybersecurity", "cbrn", "disinformation" ], "sections": [ { "heading": "CTF-Archive-Diamond", "locator": "Capability Results; Figure 2", "variantIds": [ "caisi-deepseek-v4-ctf-archive-diamond" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "SWE-Bench Verified", "locator": "Capability Results; Figure 2", "variantIds": [ "caisi-deepseek-v4-swe-verified" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "PortBench", "locator": "Capability Results; Figure 2", "variantIds": [ "caisi-deepseek-v4-portbench" ], "disposition": "variant", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "FrontierScience", "locator": "Capability Results; Figure 2", "variantIds": [ "caisi-deepseek-v4-frontierscience" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "GPQA-Diamond", "locator": "Capability Results; Figure 2", "variantIds": [ "caisi-deepseek-v4-gpqa" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "ARC-AGI-2 semi-private", "locator": "Capability Results; Figure 2", "variantIds": [ "caisi-deepseek-v4-arc-agi-2" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "OTIS-AIME-2025", "locator": "Capability Results; Figure 2", "variantIds": [ "caisi-deepseek-v4-otis-aime" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "PUMaC 2024", "locator": "Capability Results; Figure 2", "variantIds": [ "caisi-deepseek-v4-pumac" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "SMT 2025", "locator": "Capability Results; Figure 2", "variantIds": [ "caisi-deepseek-v4-smt2025" ], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "IRT aggregation and imputation", "locator": "Appendix; Figure 2 footnotes", "variantIds": [], "disposition": "limitations", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } }, { "heading": "Cost and serving configuration", "locator": "Model Serving and Inference; cost section", "variantIds": [], "disposition": "context", "note": { "en": "Protocol and result provenance are retained at this section.", "zh": "在本节保留协议与结果出处。" } } ], "reviewStatus": "section-reviewed", "scopeNote": { "en": "Reported results retain their original evaluator, configuration and measurement scope. NUWA organizes and interprets the evidence; it has not independently reproduced these experiments.", "zh": "结果保留原始执行方、配置与测量范围。NUWA 整理和解读证据，未独立复现这些实验。" }, "researchNote": { "question": { "en": "How does DeepSeek V4 Pro perform across CAISI’s selected capability tasks?", "zh": "DeepSeek V4 Pro 在 CAISI 选定能力任务上表现如何？" }, "method": { "en": "Nine task families, model-specific reasoning settings and weighted-token budgets; CTF results for DeepSeek include IRT imputation.", "zh": "九类任务、模型特定推理设置与加权 Token 预算；DeepSeek CTF 结果包含 IRT 插补。" }, "finding": { "en": "The assessment reports benchmark-specific performance and an IRT aggregate; the latter is not a frontier-risk score.", "zh": "评估报告各基准表现和 IRT 汇总；该汇总不是前沿风险分数。" }, "limits": { "en": "Controlled tasks, elicitation, benchmark familiarity and configuration differences constrain generalization to deployment.", "zh": "受控任务、诱发方式、基准熟悉程度和配置差异限制对部署的外推。" } } }, { "id": "metr-horizon-11", "title": "METR Time Horizon 1.1", "kind": "external", "sourceId": "metr-horizon-11-data", "publicationDate": null, "updatedDate": null, "modelIds": [ "claude-3-5-sonnet-june-2024", "claude-3-5-sonnet-october-2024", "claude-3-7-sonnet", "claude-3-opus", "claude-opus-4-1", "claude-opus-4", "claude-opus-4-5", "claude-opus-4-6", "gpt-3-davinci-002", "gpt-5-3-codex", "gemini-3-pro", "gemini-3-1-pro", "gpt-2", "gpt-3-5-turbo-instruct", "gpt-4", "gpt-4-november-2023", "gpt-4-turbo", "gpt-4o", "gpt-5-1-codex-max", "gpt-5-2", "gpt-5", "gpt-5-4", "o1", "o1-preview", "o3", "claude-mythos-preview-early" ], "publisherIds": [ "metr" ], "summary": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "areas": [ "autonomy" ], "sections": [ { "heading": "50% and 80% task-completion horizons", "locator": "Source YAML results; methodology page", "variantIds": [ "metr-horizon-11-protocol" ], "disposition": "linked", "note": { "en": "The data snapshot is verified on 25 October 2026. Exact first public dates of these numerical versions are not established, so historical as-of views use that conservative date.", "zh": "2026 年 10 月 25 日核验数据快照。未确定这些数值版本的首次公开日期，因此历史截止视图采用该保守日期。" } } ], "reviewStatus": "section-reviewed", "scopeNote": { "en": "The data snapshot is verified on 25 October 2026. Exact first public dates of these numerical versions are not established, so historical as-of views use that conservative date.", "zh": "2026 年 10 月 25 日核验数据快照。未确定这些数值版本的首次公开日期，因此历史截止视图采用该保守日期。" } }, { "id": "anthropic-anthropic-february-2026-risk-report", "title": "Anthropic February 2026 Risk Report", "kind": "developer", "sourceId": "official-anthropic-anthropic-february-2026-risk-report", "publicationDate": null, "updatedDate": "2026-07-08", "modelIds": [], "publisherIds": [ "anthropic" ], "summary": { "en": "Developer risk report; its scope and evidence remain those of the publishing institution.", "zh": "关于开发机构风险评估的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "section-map", "scopeNote": { "en": "Developer-authored disclosure. The section map was checked; this is not a full reading or result extraction. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。仅核查章节结构，尚未完成全文阅读或结果提取。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Autonomy and chemical/biological risk chapters", "locator": "PDF table of contents", "variantIds": [], "disposition": "context", "note": { "en": "Earlier corporate risk assessment with explicit May and July editorial revisions; retain later card/report context.", "zh": "早期公司级风险评估，含明确5月及7月修订；须保留后续模型卡/报告语境。" }, "reportedProtocols": [], "reviewDisposition": "section-map-reviewed; full-risk-argument-review-pending" } ] }, { "id": "anthropic-claude-opus-4-5-system-card", "title": "Claude Opus 4.5 System Card", "kind": "developer", "sourceId": "official-anthropic-claude-opus-4-5-system-card", "publicationDate": "2025-11-05", "updatedDate": "2025-12-05", "modelIds": [ "claude-opus-4-5" ], "publisherIds": [ "anthropic" ], "summary": { "en": "Developer documentation covering Claude Opus 4.5.", "zh": "关于Claude Opus 4.5的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "section-map", "scopeNote": { "en": "Developer-authored disclosure. The section map was checked; this is not a full reading or result extraction. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。仅核查章节结构，尚未完成全文阅读或结果提取。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "RSP, cyber and alignment sections", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "variantIds": [ "anthropic-claude-opus-4-5-system-card-cybench", "anthropic-claude-opus-4-5-system-card-cybergym", "anthropic-claude-opus-4-5-system-card-shade-arena", "anthropic-claude-opus-4-5-system-card-petri" ], "disposition": "linked", "note": { "en": "Historical comparator source located. Newer card summaries and external evaluations must preserve the tested model version, harness and safeguard configuration.", "zh": "已定位历史比较来源；新卡摘要与外部评估须保留模型版本、框架及防护配置。" }, "reportedProtocols": [ "Cybench", "CyberGym", "SHADE-Arena", "Petri" ], "reviewDisposition": "section-map-reviewed; result-level-comparison-pending" } ] }, { "id": "anthropic-claude-opus-4-6-system-card-retry", "title": "Claude Opus 4.6 System Card", "kind": "developer", "sourceId": "official-anthropic-claude-opus-4-6-system-card-retry", "publicationDate": "2026-02-06", "updatedDate": "2026-03-06", "modelIds": [ "claude-opus-4-6" ], "publisherIds": [ "anthropic" ], "summary": { "en": "Developer documentation covering Claude Opus 4.6.", "zh": "关于Claude Opus 4.6的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "section-map", "scopeNote": { "en": "Developer-authored disclosure. The section map was checked; this is not a full reading or result extraction. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。仅核查章节结构，尚未完成全文阅读或结果提取。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "RSP, agentic safety and alignment assessment", "locator": "PDF contents → named RSP, agentic safety and alignment sections", "variantIds": [ "anthropic-claude-opus-4-6-system-card-retry-cybench", "anthropic-claude-opus-4-6-system-card-retry-cybergym", "anthropic-claude-opus-4-6-system-card-retry-mask", "anthropic-claude-opus-4-6-system-card-retry-shade-arena" ], "disposition": "linked", "note": { "en": "Historical comparator for Neo, SaferAI and CAISI; safety-card configuration must be checked before importing a result.", "zh": "Neo、SaferAI及CAISI的历史比较模型；导入结果前须核对安全卡配置。" }, "reportedProtocols": [ "Cybench", "CyberGym", "MASK", "SHADE-Arena" ], "reviewDisposition": "section-map-reviewed; result-level-comparison-pending" } ] }, { "id": "anthropic-claude-opus-4-6-sabotage-risk-report", "title": "Claude Opus 4.6 Sabotage Risk Report", "kind": "developer", "sourceId": "official-anthropic-claude-opus-4-6-sabotage-risk-report", "publicationDate": "2026-02-10", "updatedDate": null, "modelIds": [ "claude-opus-4-6" ], "publisherIds": [ "anthropic" ], "summary": { "en": "Developer documentation covering Claude Opus 4.6.", "zh": "关于Claude Opus 4.6的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "autonomy", "deception" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "4 Capabilities and behaviors; 5 Risk mitigations; 6 Overall risk", "locator": "PDF §§4–6", "variantIds": [], "disposition": "context", "note": { "en": "Developer argues sabotage risk is very low but non-negligible using alignment evidence and controls. Intentional human misuse is outside this report’s principal threat model.", "zh": "开发者结合对齐证据与控制论证破坏风险很低但非零；蓄意人为滥用并非本报告主要威胁模型。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "anthropic-claude-sonnet-4-6-system-card", "title": "Claude Sonnet 4.6 System Card", "kind": "developer", "sourceId": "official-anthropic-claude-sonnet-4-6-system-card", "publicationDate": "2026-02-17", "updatedDate": "2026-03-06", "modelIds": [ "claude-sonnet-4-6" ], "publisherIds": [ "anthropic" ], "summary": { "en": "Developer documentation covering Claude Sonnet 4.6.", "zh": "关于Claude Sonnet 4.6的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "section-map", "scopeNote": { "en": "Developer-authored disclosure. The section map was checked; this is not a full reading or result extraction. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。仅核查章节结构，尚未完成全文阅读或结果提取。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "RSP, cyber and alignment sections", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "variantIds": [ "anthropic-claude-sonnet-4-6-system-card-cybench", "anthropic-claude-sonnet-4-6-system-card-cybergym", "anthropic-claude-sonnet-4-6-system-card-shade-arena", "anthropic-claude-sonnet-4-6-system-card-petri" ], "disposition": "linked", "note": { "en": "Historical comparator source located. Newer card summaries and external evaluations must preserve the tested model version, harness and safeguard configuration.", "zh": "已定位历史比较来源；新卡摘要与外部评估须保留模型版本、框架及防护配置。" }, "reportedProtocols": [ "Cybench", "CyberGym", "SHADE-Arena", "Petri" ], "reviewDisposition": "section-map-reviewed; result-level-comparison-pending" } ] }, { "id": "anthropic-claude-mythos-preview-risk-report", "title": "Alignment Risk Update: Claude Mythos Preview", "kind": "developer", "sourceId": "official-anthropic-claude-mythos-preview-risk-report", "publicationDate": "2026-04-07", "updatedDate": "2026-04-10", "modelIds": [ "claude-mythos-preview" ], "publisherIds": [ "anthropic" ], "summary": { "en": "Developer documentation covering Claude Mythos Preview.", "zh": "关于Claude Mythos Preview的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "autonomy", "deception" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "5 Evidence; 6 Risk mitigations; 10 Overall assessment", "locator": "PDF contents → Evidence, Risk mitigations, Overall assessment", "variantIds": [], "disposition": "context", "note": { "en": "A model-specific alignment risk update; subsequent cards and later corporate risk reports supersede parts of this assessment.", "zh": "模型特定的对齐风险更新；后续模型卡与公司风险报告更新了部分评估。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "anthropic-claude-opus-4-7-system-card", "title": "Claude Opus 4.7 System Card", "kind": "developer", "sourceId": "official-anthropic-claude-opus-4-7-system-card", "publicationDate": "2026-04-16", "updatedDate": null, "modelIds": [ "claude-opus-4-7" ], "publisherIds": [ "anthropic" ], "summary": { "en": "Developer documentation covering Claude Opus 4.7.", "zh": "关于Claude Opus 4.7的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "section-map", "scopeNote": { "en": "Developer-authored disclosure. The section map was checked; this is not a full reading or result extraction. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。仅核查章节结构，尚未完成全文阅读或结果提取。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "RSP, cyber and alignment sections", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "variantIds": [ "anthropic-claude-opus-4-7-system-card-cybench", "anthropic-claude-opus-4-7-system-card-cybergym", "anthropic-claude-opus-4-7-system-card-mask", "anthropic-claude-opus-4-7-system-card-shade-arena", "anthropic-claude-opus-4-7-system-card-petri" ], "disposition": "linked", "note": { "en": "Historical comparator source located. Newer card summaries and external evaluations must preserve the tested model version, harness and safeguard configuration.", "zh": "已定位历史比较来源；新卡摘要与外部评估须保留模型版本、框架及防护配置。" }, "reportedProtocols": [ "Cybench", "CyberGym", "MASK", "SHADE-Arena", "Petri" ], "reviewDisposition": "section-map-reviewed; result-level-comparison-pending" } ] }, { "id": "anthropic-claude-opus-4-8-system-card", "title": "Claude Opus 4.8 System Card", "kind": "developer", "sourceId": "official-anthropic-claude-opus-4-8-system-card", "publicationDate": "2026-05-28", "updatedDate": "2026-06-17", "modelIds": [ "claude-opus-4-8" ], "publisherIds": [ "anthropic" ], "summary": { "en": "Developer documentation covering Claude Opus 4.8.", "zh": "关于Claude Opus 4.8的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "section-map", "scopeNote": { "en": "Developer-authored disclosure. The section map was checked; this is not a full reading or result extraction. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。仅核查章节结构，尚未完成全文阅读或结果提取。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "RSP, cyber and alignment sections", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "variantIds": [ "anthropic-claude-opus-4-8-system-card-cybench", "anthropic-claude-opus-4-8-system-card-cybergym", "anthropic-claude-opus-4-8-system-card-exploitbench", "anthropic-claude-opus-4-8-system-card-mask", "anthropic-claude-opus-4-8-system-card-shade-arena", "anthropic-claude-opus-4-8-system-card-petri" ], "disposition": "linked", "note": { "en": "Historical comparator source located. Newer card summaries and external evaluations must preserve the tested model version, harness and safeguard configuration.", "zh": "已定位历史比较来源；新卡摘要与外部评估须保留模型版本、框架及防护配置。" }, "reportedProtocols": [ "Cybench", "CyberGym", "ExploitBench", "MASK", "SHADE-Arena", "Petri" ], "reviewDisposition": "section-map-reviewed; result-level-comparison-pending" } ] }, { "id": "anthropic-claude-sonnet-5-system-card", "title": "Claude Sonnet 5 System Card", "kind": "developer", "sourceId": "official-anthropic-claude-sonnet-5-system-card", "publicationDate": "2026-06-30", "updatedDate": "2026-07-10", "modelIds": [ "claude-sonnet-5" ], "publisherIds": [ "anthropic" ], "summary": { "en": "Developer documentation covering Claude Sonnet 5.", "zh": "关于Claude Sonnet 5的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "section-map", "scopeNote": { "en": "Developer-authored disclosure. The section map was checked; this is not a full reading or result extraction. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。仅核查章节结构，尚未完成全文阅读或结果提取。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "RSP, cyber and alignment sections", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "variantIds": [ "anthropic-claude-sonnet-5-system-card-cybergym", "anthropic-claude-sonnet-5-system-card-exploitbench", "anthropic-claude-sonnet-5-system-card-mask", "anthropic-claude-sonnet-5-system-card-shade-arena" ], "disposition": "linked", "note": { "en": "Historical comparator source located. Newer card summaries and external evaluations must preserve the tested model version, harness and safeguard configuration.", "zh": "已定位历史比较来源；新卡摘要与外部评估须保留模型版本、框架及防护配置。" }, "reportedProtocols": [ "CyberGym", "ExploitBench", "MASK", "SHADE-Arena" ], "reviewDisposition": "section-map-reviewed; result-level-comparison-pending" } ] }, { "id": "anthropic-anthropic-august-2026-risk-report", "title": "Anthropic August 2026 Risk Report", "kind": "developer", "sourceId": "official-anthropic-anthropic-august-2026-risk-report", "publicationDate": "2026-08-14", "updatedDate": null, "modelIds": [], "publisherIds": [ "anthropic" ], "summary": { "en": "Developer documentation covering Claude model family, internal models discussed in redacted report.", "zh": "关于Claude model family、internal models discussed in redacted report的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "autonomy", "deception", "cbrn", "cybersecurity" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "2 Autonomy threat model 1: Misalignment in high-stakes settings", "locator": "PDF printed pp.17–88", "variantIds": [ "anthropic-anthropic-august-2026-risk-report-shade-arena" ], "disposition": "linked", "note": { "en": "Structured claims, monitoring, mitigation assumptions and limitations support a low-risk developer judgment; redactions constrain independent review.", "zh": "结构化主张、监测、缓解假设与局限支撑开发者的低风险判断；删节限制独立复核。" }, "reportedProtocols": [ "SHADE-Arena", "LinuxArena" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "3 Autonomy threat model 2: Risks from automated R&D", "locator": "PDF printed pp.93–113", "variantIds": [], "disposition": "context", "note": { "en": "Separates researcher substitution, observed acceleration and forward-looking risks; capability proxies do not establish a measured economy-wide acceleration.", "zh": "区分替代研究者、已观察加速与前瞻风险；能力代理指标不能建立全经济加速结论。" }, "reportedProtocols": [ "CoBench", "AECI" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "4 Chemical and biological weapons production", "locator": "PDF printed pp.115–155", "variantIds": [], "disposition": "context", "note": { "en": "Threat models, evidence weighting and mitigations reviewed at a high level only; no hazardous task details or quantitative results retained.", "zh": "仅高层审阅威胁模型、证据权重与防护；不保留危险任务细节或数值。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "5.2 Safety process failures", "locator": "PDF printed pp.163–168", "variantIds": [], "disposition": "context", "note": { "en": "Developer discusses safety-process failures separately from capability evaluations; these are investigation/governance evidence.", "zh": "开发者将安全流程失败与能力评测区分讨论，属于调查与治理证据。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "anthropic-claude-fable-5-1-and-mythos-5-1-system-card", "title": "Claude Fable 5.1 and Mythos 5.1 System Card", "kind": "developer", "sourceId": "official-anthropic-claude-fable-5-1-and-mythos-5-1-system-card", "publicationDate": "2026-09-01", "updatedDate": null, "modelIds": [ "claude-fable-5-1", "claude-mythos-5-1" ], "publisherIds": [ "anthropic" ], "summary": { "en": "Same weights with different safeguards/access regimes. Evaluation-awareness and monitorability conclusions must preserve which configuration and snapshot was tested.", "zh": "关于Claude Fable 5.1、Claude Mythos 5.1的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cbrn", "cybersecurity", "autonomy", "deception", "disinformation" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "2 RSP evaluations", "locator": "PDF §2 (see table of contents)", "variantIds": [], "disposition": "context", "note": { "en": "Developer threshold and mitigation assessment; intrinsic capabilities, safeguarded deployment and access configurations must stay distinct.", "zh": "开发者阈值与缓解措施评估；须区分内在能力、带防护部署和访问配置。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "3 Cyber", "locator": "PDF §3", "variantIds": [ "anthropic-claude-fable-5-1-and-mythos-5-1-system-card-exploitbench", "anthropic-claude-fable-5-1-and-mythos-5-1-system-card-exploitgym" ], "disposition": "linked", "note": { "en": "Cyber capability tests and safeguards have different settings; the report does not establish unrestricted real-world attack success.", "zh": "网络能力测试与防护测试配置不同；不构成现实中无限制攻击成功的证据。" }, "reportedProtocols": [ "ExploitBench", "OSS-Fuzz", "ExploitGym", "Firefox 147" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "5 Agentic safety", "locator": "PDF §5", "variantIds": [], "disposition": "context", "note": { "en": "Malicious use and prompt-injection tests cover coding, browser and computer-use surfaces; internal and external tests are distinguished.", "zh": "恶意使用与提示注入测试涵盖代码、浏览器与计算机操作；保留内部和外部测试差异。" }, "reportedProtocols": [ "Indirect Prompt Injection (IPI)" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "6 Alignment assessment", "locator": "PDF §6", "variantIds": [ "anthropic-claude-fable-5-1-and-mythos-5-1-system-card-mask", "anthropic-claude-fable-5-1-and-mythos-5-1-system-card-shade-arena" ], "disposition": "linked", "note": { "en": "Behavioral audits, honesty, oversight evasion and evaluation-awareness evidence are bounded by artificial scenarios and available reasoning traces.", "zh": "行为审计、诚实性、规避监督及评测意识证据受模拟场景与可见推理轨迹限制。" }, "reportedProtocols": [ "MASK", "SHADE-Arena", "LinuxArena" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "anthropic-claude-opus-5-5-system-card", "title": "Claude Opus 5.5 System Card", "kind": "developer", "sourceId": "official-anthropic-claude-opus-5-5-system-card", "publicationDate": "2026-09-22", "updatedDate": null, "modelIds": [ "claude-opus-5-5" ], "publisherIds": [ "anthropic" ], "summary": { "en": "Developer reports broad alignment improvements alongside regressions involving pasted instructions and unverifiable authorization. Pilot monitoring found no long-horizon strategic deception; rare concerning behavior and assessment blind spots remain.", "zh": "关于Claude Opus 5.5的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cbrn", "cybersecurity", "autonomy", "deception", "disinformation" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "2 RSP evaluations", "locator": "PDF §2 (see table of contents)", "variantIds": [], "disposition": "context", "note": { "en": "Developer threshold and mitigation assessment; intrinsic capabilities, safeguarded deployment and access configurations must stay distinct.", "zh": "开发者阈值与缓解措施评估；须区分内在能力、带防护部署和访问配置。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "3 Cyber", "locator": "PDF §3", "variantIds": [ "anthropic-claude-opus-5-5-system-card-exploitbench", "anthropic-claude-opus-5-5-system-card-exploitgym" ], "disposition": "linked", "note": { "en": "Cyber capability tests and safeguards have different settings; the report does not establish unrestricted real-world attack success.", "zh": "网络能力测试与防护测试配置不同；不构成现实中无限制攻击成功的证据。" }, "reportedProtocols": [ "ExploitBench", "CyScenarioBench", "Binary Exploitation Benchmark", "OSS-Fuzz", "ExploitGym" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "5 Agentic safety", "locator": "PDF §5", "variantIds": [], "disposition": "context", "note": { "en": "Malicious use and prompt-injection tests cover coding, browser and computer-use surfaces; internal and external tests are distinguished.", "zh": "恶意使用与提示注入测试涵盖代码、浏览器与计算机操作；保留内部和外部测试差异。" }, "reportedProtocols": [ "Indirect Prompt Injection (IPI)" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "6 Alignment assessment", "locator": "PDF §6", "variantIds": [ "anthropic-claude-opus-5-5-system-card-mask", "anthropic-claude-opus-5-5-system-card-shade-arena" ], "disposition": "linked", "note": { "en": "Behavioral audits, honesty, oversight evasion and evaluation-awareness evidence are bounded by artificial scenarios and available reasoning traces.", "zh": "行为审计、诚实性、规避监督及评测意识证据受模拟场景与可见推理轨迹限制。" }, "reportedProtocols": [ "MASK", "SHADE-Arena", "LinuxArena" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "anthropic-claude-sonnet-5-5-system-card", "title": "Claude Sonnet 5.5 System Card", "kind": "developer", "sourceId": "official-anthropic-claude-sonnet-5-5-system-card", "publicationDate": "2026-09-28", "updatedDate": null, "modelIds": [ "claude-sonnet-5-5" ], "publisherIds": [ "anthropic" ], "summary": { "en": "Condensed non-frontier card; broad capabilities are bounded against Opus 5.5, rather than repeating every high-effort frontier assessment.", "zh": "关于Claude Sonnet 5.5的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cbrn", "cybersecurity", "autonomy", "deception", "disinformation" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "2 RSP evaluations", "locator": "PDF §2 (see table of contents)", "variantIds": [], "disposition": "context", "note": { "en": "Developer threshold and mitigation assessment; intrinsic capabilities, safeguarded deployment and access configurations must stay distinct.", "zh": "开发者阈值与缓解措施评估；须区分内在能力、带防护部署和访问配置。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "3 Cyber", "locator": "PDF §3", "variantIds": [ "anthropic-claude-sonnet-5-5-system-card-exploitbench", "anthropic-claude-sonnet-5-5-system-card-exploitgym" ], "disposition": "linked", "note": { "en": "Cyber capability tests and safeguards have different settings; the report does not establish unrestricted real-world attack success.", "zh": "网络能力测试与防护测试配置不同；不构成现实中无限制攻击成功的证据。" }, "reportedProtocols": [ "ExploitBench", "CyScenarioBench", "Binary Exploitation Benchmark", "OSS-Fuzz", "ExploitGym" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "5 Agentic safety", "locator": "PDF §5", "variantIds": [], "disposition": "context", "note": { "en": "Malicious use and prompt-injection tests cover coding, browser and computer-use surfaces; internal and external tests are distinguished.", "zh": "恶意使用与提示注入测试涵盖代码、浏览器与计算机操作；保留内部和外部测试差异。" }, "reportedProtocols": [ "Indirect Prompt Injection (IPI)" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "6 Alignment assessment", "locator": "PDF §6", "variantIds": [ "anthropic-claude-sonnet-5-5-system-card-mask", "anthropic-claude-sonnet-5-5-system-card-shade-arena" ], "disposition": "linked", "note": { "en": "Behavioral audits, honesty, oversight evasion and evaluation-awareness evidence are bounded by artificial scenarios and available reasoning traces.", "zh": "行为审计、诚实性、规避监督及评测意识证据受模拟场景与可见推理轨迹限制。" }, "reportedProtocols": [ "MASK", "SHADE-Arena", "LinuxArena" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "anthropic-claude-haiku-5-5-system-card", "title": "Claude Haiku 5.5 System Card", "kind": "developer", "sourceId": "official-anthropic-claude-haiku-5-5-system-card", "publicationDate": "2026-10-07", "updatedDate": null, "modelIds": [ "claude-haiku-5-5" ], "publisherIds": [ "anthropic" ], "summary": { "en": "Condensed card explicitly omits some human-intensive evaluations because this is not a frontier model; do not infer that missing evaluations passed.", "zh": "关于Claude Haiku 5.5的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cbrn", "cybersecurity", "autonomy", "deception", "disinformation" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "2 RSP evaluations", "locator": "PDF §2 (see table of contents)", "variantIds": [], "disposition": "context", "note": { "en": "Developer threshold and mitigation assessment; intrinsic capabilities, safeguarded deployment and access configurations must stay distinct.", "zh": "开发者阈值与缓解措施评估；须区分内在能力、带防护部署和访问配置。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "3 Cyber", "locator": "PDF §3", "variantIds": [ "anthropic-claude-haiku-5-5-system-card-exploitbench", "anthropic-claude-haiku-5-5-system-card-exploitgym" ], "disposition": "linked", "note": { "en": "Cyber capability tests and safeguards have different settings; the report does not establish unrestricted real-world attack success.", "zh": "网络能力测试与防护测试配置不同；不构成现实中无限制攻击成功的证据。" }, "reportedProtocols": [ "ExploitBench", "CyScenarioBench", "Binary Exploitation Benchmark", "OSS-Fuzz", "ExploitGym" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "5 Agentic safety", "locator": "PDF §5", "variantIds": [], "disposition": "context", "note": { "en": "Malicious use and prompt-injection tests cover coding, browser and computer-use surfaces; internal and external tests are distinguished.", "zh": "恶意使用与提示注入测试涵盖代码、浏览器与计算机操作；保留内部和外部测试差异。" }, "reportedProtocols": [ "Indirect Prompt Injection (IPI)" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "6 Alignment assessment", "locator": "PDF §6", "variantIds": [ "anthropic-claude-haiku-5-5-system-card-mask", "anthropic-claude-haiku-5-5-system-card-shade-arena" ], "disposition": "linked", "note": { "en": "Behavioral audits, honesty, oversight evasion and evaluation-awareness evidence are bounded by artificial scenarios and available reasoning traces.", "zh": "行为审计、诚实性、规避监督及评测意识证据受模拟场景与可见推理轨迹限制。" }, "reportedProtocols": [ "MASK", "SHADE-Arena", "LinuxArena" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "deepseek-deepseek-v4-pro-model-card", "title": "DeepSeek-V4-Pro Model Card", "kind": "developer", "sourceId": "official-deepseek-deepseek-v4-pro-model-card", "publicationDate": null, "updatedDate": null, "modelIds": [ "deepseek-v4-pro" ], "publisherIds": [ "deepseek" ], "summary": { "en": "Developer documentation covering DeepSeek-V4-Pro.", "zh": "关于DeepSeek-V4-Pro的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Evaluation Results", "locator": "Original source → Evaluation Results", "variantIds": [], "disposition": "context", "note": { "en": "Capability table and deployment guidance do not substitute for CAISI or Neo independent risk evaluations.", "zh": "能力表与部署说明不能替代CAISI或Neo独立风险评估。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established" } ] }, { "id": "deepseek-deepseek-v4-towards-highly-efficient-million-token-context-intelligence", "title": "DeepSeek-V4: Towards Highly Efficient Million-Token Context Intelligence", "kind": "developer", "sourceId": "official-deepseek-deepseek-v4-towards-highly-efficient-million-token-context-intelligence", "publicationDate": null, "updatedDate": null, "modelIds": [ "deepseek-v4-pro", "deepseek-v4-flash" ], "publisherIds": [ "deepseek" ], "summary": { "en": "Developer documentation covering DeepSeek-V4-Pro, DeepSeek-V4-Flash.", "zh": "关于DeepSeek-V4-Pro、DeepSeek-V4-Flash的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "5 Post-Training; 6 Conclusion, Limitations, and Future Directions", "locator": "Original source → 5 Post-Training; 6 Conclusion, Limitations, and Future Directions", "variantIds": [], "disposition": "context", "note": { "en": "Architecture, training and general capabilities dominate; limitations include task/reasoning shortcomings. No dedicated frontier-risk assessment located.", "zh": "重点为架构、训练与一般能力；局限包含任务与推理不足，未发现独立前沿风险评估。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established" } ] }, { "id": "google-deepmind-gemini-4-argon-safety-security-report", "title": "Gemini 4 Argon Safety & Security Report", "kind": "developer", "sourceId": "official-google-deepmind-gemini-4-argon-safety-security-report", "publicationDate": null, "updatedDate": null, "modelIds": [ "gemini-4-argon" ], "publisherIds": [ "deepmind" ], "summary": { "en": "Developer documentation covering Gemini 4 Argon.", "zh": "关于Gemini 4 Argon的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cbrn", "cybersecurity", "autonomy", "deception", "disinformation" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "3.2 Misuse risk assessment", "locator": "PDF printed pp.21–53", "variantIds": [ "google-deepmind-gemini-4-argon-safety-security-report-exploitbench" ], "disposition": "linked", "note": { "en": "Separates inherent capability from residual risk and uses internal and independent tests; no operational biology details or numerical scores are transcribed.", "zh": "区分内在能力与剩余风险，结合内部及独立测试；未转录生物操作细节或数值。" }, "reportedProtocols": [ "ExploitBench" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "3.3 Misuse Safeguards", "locator": "PDF printed pp.55–62", "variantIds": [], "disposition": "context", "note": { "en": "Coverage and adversarial robustness are assessed separately; safeguards are part of the deployment risk argument.", "zh": "分别评估覆盖度与对抗稳健性；防护措施是部署风险论证的组成部分。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "3.4 Loss of Control", "locator": "PDF printed pp.63–155", "variantIds": [], "disposition": "context", "note": { "en": "Holistic safety argument combines capabilities, propensities, monitoring and security assumptions; conclusions depend on those stated assumptions.", "zh": "综合安全论证结合能力、倾向、监测与安全假设；结论依赖所述假设。" }, "reportedProtocols": [ "Gram", "LinuxArena" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "3.4.5 Chain-of-Thought Transparency Assessment", "locator": "PDF printed pp.112–155", "variantIds": [], "disposition": "context", "note": { "en": "Developer reports more difficult monitorability than prior models; complex covert-task results are bounded by task difficulty and monitor strength.", "zh": "开发者称可监测性较前代更困难；复杂隐蔽任务结论受任务难度与监测器强度限制。" }, "reportedProtocols": [ "CoT controllability", "monitor evasion" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "3.5 Evaluation Awareness and Behavior Effects", "locator": "PDF printed pp.156–158", "variantIds": [], "disposition": "context", "note": { "en": "Awareness can change compliance and local grader behavior. Audits found no confirmed strategic underperformance; this does not rule out unspoken awareness.", "zh": "评测意识可改变合规及局部评分行为；审计未确认战略性低报能力，但不排除未表述的意识。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "6 Guarding users against harm from third parties", "locator": "PDF §6", "variantIds": [], "disposition": "context", "note": { "en": "Prompt-injection resistance is a separate model/system robustness finding.", "zh": "提示注入抵抗力属于独立的模型/系统稳健性结论。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "google-deepmind-gemini-4-argon-model-card", "title": "Gemini 4 Argon Model Card", "kind": "developer", "sourceId": "official-google-deepmind-gemini-4-argon-model-card", "publicationDate": "2026-10-13", "updatedDate": "2026-10-13", "modelIds": [ "gemini-4-argon" ], "publisherIds": [ "deepmind" ], "summary": { "en": "Developer documentation covering Gemini 4 Argon.", "zh": "关于Gemini 4 Argon的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cybersecurity" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Frontier Safety Assessment", "locator": "Model card → Safety & Security → Frontier Safety Assessment", "variantIds": [], "disposition": "context", "note": { "en": "Cannot rule out cyber and biological critical capability levels or the stealth/situational-awareness tracked level; residual-risk arguments rely on mitigations.", "zh": "无法排除网络、生物关键能力级别及隐蔽性/情境意识跟踪级别；剩余风险论证依赖缓解措施。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "meta-muse-glimmer-model-card", "title": "Muse Glimmer Model Card", "kind": "developer", "sourceId": "official-meta-muse-glimmer-model-card", "publicationDate": null, "updatedDate": null, "modelIds": [ "muse-glimmer" ], "publisherIds": [ "meta" ], "summary": { "en": "Developer documentation covering Muse Glimmer.", "zh": "关于Muse Glimmer的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Trust and Safety; Preparedness; Train-Time Mitigations", "locator": "Original source → Trust and Safety; Preparedness; Train-Time Mitigations", "variantIds": [], "disposition": "context", "note": { "en": "Developer says this smaller model falls outside its frontier-model definition, while still assessing preparedness and local-agent risks. Do not present it as a frontier flagship.", "zh": "开发者称该小模型不属于其前沿模型定义，但仍评估准备度及本地代理风险；不应呈现为前沿旗舰。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "meta-muse-spark-1-2-evaluation-report", "title": "Muse Spark 1.2 Multimodal Evaluation Methodology", "kind": "developer", "sourceId": "official-meta-muse-spark-1-2-evaluation-report", "publicationDate": null, "updatedDate": null, "modelIds": [ "muse-spark-1-2" ], "publisherIds": [ "meta" ], "summary": { "en": "Developer documentation covering Muse Spark 1.2.", "zh": "关于Muse Spark 1.2的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Overall Methodology", "locator": "Original source → Overall Methodology", "variantIds": [], "disposition": "context", "note": { "en": "Retrieved document is multimodal capability methodology, with anti-cheating restrictions; it is not a comprehensive frontier-risk report.", "zh": "取得文档为多模态能力评估方法，含防作弊限制，并非完整前沿风险报告。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "meta-muse-spark-1-3-evaluation-report", "title": "Muse Spark 1.3 Evaluation Methodology", "kind": "developer", "sourceId": "official-meta-muse-spark-1-3-evaluation-report", "publicationDate": null, "updatedDate": null, "modelIds": [ "muse-spark-1-3" ], "publisherIds": [ "meta" ], "summary": { "en": "Developer documentation covering Muse Spark 1.3.", "zh": "关于Muse Spark 1.3的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Evaluation Methodology", "locator": "Original source → Evaluation Methodology", "variantIds": [], "disposition": "context", "note": { "en": "Retrieved document primarily describes capability-evaluation methodology; keep separate from full safety and preparedness reports.", "zh": "取得文档主要描述能力评估方法，须与完整安全与准备度报告区分。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "meta-muse-spark-contemplating-safety-preparedness-report", "title": "Muse Spark Contemplating Safety & Preparedness Report", "kind": "developer", "sourceId": "official-meta-muse-spark-contemplating-safety-preparedness-report", "publicationDate": null, "updatedDate": null, "modelIds": [ "muse-spark-contemplating" ], "publisherIds": [ "meta" ], "summary": { "en": "Developer documentation covering Muse Spark Contemplating.", "zh": "关于Muse Spark Contemplating的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Preparedness and behavior assessment", "locator": "Original source → Preparedness and behavior assessment", "variantIds": [], "disposition": "context", "note": { "en": "Separate deployment configuration/report; conclusions should not be pooled with baseline Muse Spark or later API-enabled releases.", "zh": "独立部署配置及报告；结论不能与基础Muse Spark或后续开放API版本合并。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "meta-muse-spark-open-evaluation-methodology", "title": "Muse Spark Open Evaluation Methodology", "kind": "developer", "sourceId": "official-meta-muse-spark-open-evaluation-methodology", "publicationDate": null, "updatedDate": null, "modelIds": [ "muse-spark-open" ], "publisherIds": [ "meta" ], "summary": { "en": "Developer documentation covering Muse Spark Open.", "zh": "关于Muse Spark Open的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Overall Methodology; safety benchmark descriptions", "locator": "Original source → Overall Methodology; safety benchmark descriptions", "variantIds": [ "meta-muse-spark-open-evaluation-methodology-fortress" ], "disposition": "linked", "note": { "en": "Methodology distinguishes benchmark-provider results, developer reruns and launch-post self-reports; comparisons are not uniformly matched experiments.", "zh": "方法说明区分基准机构结果、开发者复测与发布自报，比较并非统一匹配实验。" }, "reportedProtocols": [ "FORTRESS" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "meta-muse-spark-open-model-card", "title": "Muse Spark Open Model Card", "kind": "developer", "sourceId": "official-meta-muse-spark-open-model-card", "publicationDate": null, "updatedDate": null, "modelIds": [ "muse-spark-open" ], "publisherIds": [ "meta" ], "summary": { "en": "Developer documentation covering Muse Spark Open.", "zh": "关于Muse Spark Open的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Safety and Preparedness; Considerations and Limitations", "locator": "Original source → Safety and Preparedness; Considerations and Limitations", "variantIds": [ "meta-muse-spark-open-model-card-fortress" ], "disposition": "linked", "note": { "en": "Open-weight card covers content safety, agentic actions, privacy and preparedness. Its additional post-training does not recreate hosted system guardrails; fuller evaluation report was forthcoming.", "zh": "开放权重卡涵盖内容、代理行为、隐私与准备度；额外后训练不能替代托管系统防护，更完整报告仍待发布。" }, "reportedProtocols": [ "FORTRESS", "StrongREJECT v2" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "meta-muse-spark-safety-preparedness-report-meta-original", "title": "Muse Spark Safety & Preparedness Report (Meta-hosted May 26 revision)", "kind": "developer", "sourceId": "official-meta-muse-spark-safety-preparedness-report-meta-original", "publicationDate": null, "updatedDate": "2026-05-26", "modelIds": [ "muse-spark" ], "publisherIds": [ "meta" ], "summary": { "en": "Developer documentation covering Muse Spark.", "zh": "关于Muse Spark的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Change Log and 4.1.1 Instruction Hierarchy", "locator": "PDF front-matter change log and §4.1.1", "variantIds": [], "disposition": "context", "note": { "en": "This Meta-hosted revision corrects an instruction-hierarchy implementation bug; the report says the assessment was unchanged. It differs from the later arXiv HTML revision.", "zh": "该Meta托管修订纠正指令层级测试实现问题，报告称评估未变；与后续arXiv HTML修订不同。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "meta-muse-wonder-model-card", "title": "Muse Wonder Model Card", "kind": "developer", "sourceId": "official-meta-muse-wonder-model-card", "publicationDate": null, "updatedDate": "2026-10-23", "modelIds": [ "muse-wonder", "muse-wonder-full" ], "publisherIds": [ "meta" ], "summary": { "en": "Developer documentation covering Muse Wonder, Muse Wonder Full.", "zh": "关于Muse Wonder、Muse Wonder Full的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cbrn", "cybersecurity", "autonomy", "deception" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "4 Risk assessments under the Meta Superintelligence Scaling Framework", "locator": "PDF pp.7–10", "variantIds": [ "meta-muse-wonder-model-card-cybergym", "meta-muse-wonder-model-card-exploitbench", "meta-muse-wonder-model-card-exploitgym", "meta-muse-wonder-model-card-mask", "meta-muse-wonder-model-card-agentic-misalignment" ], "disposition": "linked", "note": { "en": "Most capable Full variant bounds family assessment. Developer does not rule out high unmitigated biochemical capability; its cyber and loss-of-control judgment is moderate or lower under its own framework.", "zh": "以最强Full变体评估整个系列；开发者不排除缓解前高生化能力，并在自身框架下评为网络及失控风险中等或更低。" }, "reportedProtocols": [ "CyberGym", "ExploitBench", "ExploitGym", "MASK", "Agentic Misalignment" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "Updated ExploitGym Evaluations", "locator": "PDF p.7, note dated October 23, 2026", "variantIds": [ "meta-muse-wonder-model-card-exploitgym" ], "disposition": "linked", "note": { "en": "Corrected harness and intended-flag reporting changed the published result. Use this revision, not the earlier value, for any future comparison.", "zh": "修正测试框架与预期标志计数后公布结果发生变化；未来比较应使用本修订而非旧值。" }, "reportedProtocols": [ "ExploitGym" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "5 Trust and Safety; 6 Considerations and Limitations", "locator": "PDF pp.11–12", "variantIds": [], "disposition": "context", "note": { "en": "Card states that tests cannot cover all situations and recommends context-specific evaluation and additional system guardrails. Full safety report remained described as forthcoming in release copy.", "zh": "卡片说明测试无法覆盖所有场景，建议按用例评估并增加系统防护；发布文案仍称完整安全报告即将发布。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "meta-muse-spark-safety-preparedness-report", "title": "Muse Spark Safety & Preparedness Report", "kind": "developer", "sourceId": "official-meta-muse-spark-safety-preparedness-report", "publicationDate": "2026-05-14", "updatedDate": "2026-08-24", "modelIds": [ "muse-spark" ], "publisherIds": [ "meta" ], "summary": { "en": "Developer documentation covering Muse Spark.", "zh": "关于Muse Spark的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cbrn", "cybersecurity", "autonomy", "deception" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "2.1 Chemical & Biological Risks", "locator": "https://arxiv.org/html/2606.12429#S2.SS1", "variantIds": [], "disposition": "context", "note": { "en": "Developer separates high pre-mitigation capability from residual risk of the Meta AI deployment; no numerical biology results or task content extracted.", "zh": "开发者区分缓解前高能力与Meta AI部署的剩余风险；未提取生物数值或任务内容。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "2.2 Cybersecurity", "locator": "https://arxiv.org/html/2606.12429#S2.SS2", "variantIds": [ "meta-muse-spark-safety-preparedness-report-cybench", "meta-muse-spark-safety-preparedness-report-cybergym" ], "disposition": "linked", "note": { "en": "Covers knowledge, agentic capability, refusals and insecure-code propensity; public and private tasks have different limits.", "zh": "涵盖知识、代理能力、拒绝与不安全代码倾向；公开和私有任务的推论限制不同。" }, "reportedProtocols": [ "Cybench", "CyberGym", "CyScenarioBench", "CyberSecEval" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "2.3 Loss of Control", "locator": "https://arxiv.org/html/2606.12429#S2.SS3", "variantIds": [ "meta-muse-spark-safety-preparedness-report-mask", "meta-muse-spark-safety-preparedness-report-agentic-misalignment" ], "disposition": "linked", "note": { "en": "Assessment integrates evaluation reliability, monitorability and propensities; future-facing agentic tests exceed the initial Meta AI deployment context.", "zh": "综合评测可靠性、可监测性与倾向；前瞻代理测试超出最初Meta AI部署场景。" }, "reportedProtocols": [ "MASK", "Agentic Misalignment" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "3.2 Agent Robustness", "locator": "https://arxiv.org/html/2606.12429#S3.SS2", "variantIds": [ "meta-muse-spark-safety-preparedness-report-agentdojo" ], "disposition": "linked", "note": { "en": "Indirect prompt injection tests use benign users and realistic scaffolds; residual attack susceptibility remains.", "zh": "间接提示注入测试使用善意用户与真实化框架；仍存在攻击脆弱性。" }, "reportedProtocols": [ "SIREN", "AgentDojo" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "4.1.5 Honesty", "locator": "https://arxiv.org/html/2606.12429#S4.SS1.SSS5", "variantIds": [ "meta-muse-spark-safety-preparedness-report-mask", "meta-muse-spark-safety-preparedness-report-deceptionbench" ], "disposition": "linked", "note": { "en": "Distinguishes known-belief dishonesty from hallucination and third-party deceptive assistance.", "zh": "区分明知事实的失实陈述、幻觉和协助欺骗第三方。" }, "reportedProtocols": [ "MASK", "DeceptionBench" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "4.1.10 Scheming and 4.1.11 Evaluation Awareness", "locator": "https://arxiv.org/html/2606.12429#S4.SS1.SSS11", "variantIds": [ "meta-muse-spark-safety-preparedness-report-alignment-faking", "meta-muse-spark-safety-preparedness-report-agentic-misalignment" ], "disposition": "linked", "note": { "en": "Apollo observed unusually high evaluation awareness. Follow-up behavioral impact was limited to a subset and did not inform hazardous-capability launch decisions; that does not prove no awareness effect.", "zh": "Apollo观察到高评测意识；复核发现行为影响限于部分测试且未涉及危险能力发布决策，但不能据此证明没有意识效应。" }, "reportedProtocols": [ "Alignment Faking", "Agentic Misalignment" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "meta-muse-spark-1-1-evaluation-report", "title": "Muse Spark 1.1 Evaluation Report", "kind": "developer", "sourceId": "official-meta-muse-spark-1-1-evaluation-report", "publicationDate": "2026-07-09", "updatedDate": null, "modelIds": [ "muse-spark-1-1" ], "publisherIds": [ "meta" ], "summary": { "en": "Developer documentation covering Muse Spark 1.1.", "zh": "关于Muse Spark 1.1的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cbrn", "cybersecurity", "autonomy", "deception" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "2 Preparedness Assessment", "locator": "PDF §§2.1–2.3", "variantIds": [ "meta-muse-spark-1-1-evaluation-report-cybench", "meta-muse-spark-1-1-evaluation-report-cybergym", "meta-muse-spark-1-1-evaluation-report-exploitgym", "meta-muse-spark-1-1-evaluation-report-shade-arena", "meta-muse-spark-1-1-evaluation-report-mask" ], "disposition": "linked", "note": { "en": "API agentic affordances expand the assessment context. Developer cannot rule out high pre-mitigation cyber and biochemical capability; residual-risk judgment includes safeguards.", "zh": "API代理能力扩大评估范围；开发者无法排除缓解前高网络与生化能力，剩余风险判断包含防护。" }, "reportedProtocols": [ "Cybench", "CyberGym", "ExploitGym", "CyScenarioBench", "SHADE-Arena", "MASK" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "3 Adversarial Robustness", "locator": "PDF §3", "variantIds": [ "meta-muse-spark-1-1-evaluation-report-fortress", "meta-muse-spark-1-1-evaluation-report-agentdojo" ], "disposition": "linked", "note": { "en": "Direct misuse, prompt injection and false refusals are evaluated separately.", "zh": "分别评估直接滥用、提示注入与错误拒绝。" }, "reportedProtocols": [ "StrongREJECT v2", "FORTRESS", "AgentHarm Verified", "AgentDojo", "SWE-PI", "GraySwan ART" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "4 Model Behavior", "locator": "PDF §4", "variantIds": [ "meta-muse-spark-1-1-evaluation-report-mask", "meta-muse-spark-1-1-evaluation-report-deceptionbench", "meta-muse-spark-1-1-evaluation-report-agentic-misalignment" ], "disposition": "linked", "note": { "en": "Honesty, risk escalation, evaluation awareness and automated alignment audits are distinct behavior constructs; artificial scenario results have limited deployment generality.", "zh": "诚实性、风险升级、评测意识与自动对齐审计属于不同构念；模拟场景向部署泛化有限。" }, "reportedProtocols": [ "SAVE-Bench", "MASK", "DeceptionBench", "Agentic Misalignment" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "mistral-shieldstral", "title": "Shieldstral", "kind": "developer", "sourceId": "official-mistral-shieldstral", "publicationDate": null, "updatedDate": null, "modelIds": [], "publisherIds": [ "mistral" ], "summary": { "en": "Developer documentation covering Shieldstral.", "zh": "关于Shieldstral的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Safety classification, refusal detection and multimodal safety evaluations", "locator": "Original source → Safety classification, refusal detection and multimodal safety evaluations", "variantIds": [ "mistral-shieldstral-harmbench" ], "disposition": "linked", "note": { "en": "Policy-adaptive safety classifier; its moderation results do not establish the safety of Mistral Large 4.", "zh": "策略自适应安全分类器；其审核结果不能证明Mistral Large 4安全。" }, "reportedProtocols": [ "WildGuardTest", "HarmBench", "XSTest" ], "reviewDisposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established" } ] }, { "id": "moonshot-kimi-k3-technical-report", "title": "Kimi K3 Technical Report", "kind": "developer", "sourceId": "official-moonshot-kimi-k3-technical-report", "publicationDate": null, "updatedDate": null, "modelIds": [ "kimi-k3" ], "publisherIds": [ "moonshot" ], "summary": { "en": "Developer documentation covering Kimi K3.", "zh": "关于Kimi K3的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "4.2 Agentic Reinforcement Learning; 5 training infrastructure", "locator": "Original source → 4.2 Agentic Reinforcement Learning; 5 training infrastructure", "variantIds": [], "disposition": "context", "note": { "en": "Training discusses hidden verifiers and sandbox isolation. The reviewed report contains no dedicated catastrophic-risk assessment; Neo findings remain independent sources.", "zh": "训练部分讨论隐藏验证器与沙箱隔离；所审报告没有独立的灾难风险评估章节，Neo结论仍属独立来源。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established" } ] }, { "id": "openai-update-to-gpt-5-system-card-gpt-5-2", "title": "Update to GPT-5 System Card: GPT-5.2", "kind": "developer", "sourceId": "official-openai-update-to-gpt-5-system-card-gpt-5-2", "publicationDate": "2025-12-11", "updatedDate": null, "modelIds": [ "gpt-5", "gpt-5-2" ], "publisherIds": [ "openai" ], "summary": { "en": "Developer documentation covering GPT-5, GPT-5.2.", "zh": "关于GPT-5、GPT-5.2的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cybersecurity", "deception" ], "reviewStatus": "section-map", "scopeNote": { "en": "Developer-authored disclosure. The section map was checked; this is not a full reading or result extraction. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。仅核查章节结构，尚未完成全文阅读或结果提取。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Preparedness, alignment and monitorability", "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections", "variantIds": [ "openai-update-to-gpt-5-system-card-gpt-5-2-mle-bench" ], "disposition": "linked", "note": { "en": "Comparator evaluation sections located; capability, refusal, monitoring and deployment safeguards are separate constructs. No numerical evidence is imported.", "zh": "已定位比较评估章节；能力、拒绝、监测及部署防护为不同构念，未导入数值证据。" }, "reportedProtocols": [ "Cyber range", "MLE-bench" ], "reviewDisposition": "section-map-reviewed; result-level-comparison-pending" } ] }, { "id": "openai-gpt-5-4-thinking-system-card", "title": "GPT-5.4 Thinking System Card", "kind": "developer", "sourceId": "official-openai-gpt-5-4-thinking-system-card", "publicationDate": "2026-03-05", "updatedDate": null, "modelIds": [ "gpt-5-4-thinking" ], "publisherIds": [ "openai" ], "summary": { "en": "Developer documentation covering GPT-5.4 Thinking.", "zh": "关于GPT-5.4 Thinking的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cybersecurity", "deception" ], "reviewStatus": "section-map", "scopeNote": { "en": "Developer-authored disclosure. The section map was checked; this is not a full reading or result extraction. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。仅核查章节结构，尚未完成全文阅读或结果提取。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Preparedness, alignment and monitorability", "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections", "variantIds": [ "openai-gpt-5-4-thinking-system-card-mle-bench" ], "disposition": "linked", "note": { "en": "Comparator evaluation sections located; capability, refusal, monitoring and deployment safeguards are separate constructs. No numerical evidence is imported.", "zh": "已定位比较评估章节；能力、拒绝、监测及部署防护为不同构念，未导入数值证据。" }, "reportedProtocols": [ "Cyber range", "SWE-bench Verified", "MLE-bench" ], "reviewDisposition": "section-map-reviewed; result-level-comparison-pending" } ] }, { "id": "openai-gpt-5-5-system-card", "title": "GPT-5.5 System Card", "kind": "developer", "sourceId": "official-openai-gpt-5-5-system-card", "publicationDate": "2026-04-23", "updatedDate": null, "modelIds": [ "gpt-5-5" ], "publisherIds": [ "openai" ], "summary": { "en": "Developer documentation covering GPT-5.5.", "zh": "关于GPT-5.5的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cybersecurity", "deception" ], "reviewStatus": "section-map", "scopeNote": { "en": "Developer-authored disclosure. The section map was checked; this is not a full reading or result extraction. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。仅核查章节结构，尚未完成全文阅读或结果提取。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Preparedness, alignment and monitorability", "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections", "variantIds": [ "openai-gpt-5-5-system-card-mle-bench" ], "disposition": "linked", "note": { "en": "Comparator evaluation sections located; capability, refusal, monitoring and deployment safeguards are separate constructs. No numerical evidence is imported.", "zh": "已定位比较评估章节；能力、拒绝、监测及部署防护为不同构念，未导入数值证据。" }, "reportedProtocols": [ "Cyber range", "SWE-bench Verified", "MLE-bench" ], "reviewDisposition": "section-map-reviewed; result-level-comparison-pending" } ] }, { "id": "openai-gpt-5-6-system-card", "title": "GPT-5.6 System Card", "kind": "developer", "sourceId": "official-openai-gpt-5-6-system-card", "publicationDate": "2026-07-09", "updatedDate": null, "modelIds": [ "gpt-5-6" ], "publisherIds": [ "openai" ], "summary": { "en": "Developer documentation covering GPT-5.6.", "zh": "关于GPT-5.6的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cbrn", "cybersecurity", "deception" ], "reviewStatus": "section-map", "scopeNote": { "en": "Developer-authored disclosure. The section map was checked; this is not a full reading or result extraction. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。仅核查章节结构，尚未完成全文阅读或结果提取。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Preparedness, alignment and monitorability", "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections", "variantIds": [ "openai-gpt-5-6-system-card-cybergym", "openai-gpt-5-6-system-card-exploitbench", "openai-gpt-5-6-system-card-mle-bench" ], "disposition": "linked", "note": { "en": "Comparator evaluation sections located; capability, refusal, monitoring and deployment safeguards are separate constructs. No numerical evidence is imported.", "zh": "已定位比较评估章节；能力、拒绝、监测及部署防护为不同构念，未导入数值证据。" }, "reportedProtocols": [ "CyberGym", "ExploitBench", "Cyber range", "SWE-bench Verified", "MLE-bench" ], "reviewDisposition": "section-map-reviewed; result-level-comparison-pending" } ] }, { "id": "openai-gpt-6-astra-system-card", "title": "GPT-6 Astra System Card", "kind": "developer", "sourceId": "official-openai-gpt-6-astra-system-card", "publicationDate": "2026-09-03", "updatedDate": "2026-09-09", "modelIds": [ "gpt-6-astra", "gpt-6-sol-september", "gpt-6-luna-september" ], "publisherIds": [ "openai" ], "summary": { "en": "Developer documentation covering GPT-6 Astra, GPT-6 Sol (September), GPT-6 Luna (September), dots.", "zh": "关于GPT-6 Astra、GPT-6 Sol (September)、GPT-6 Luna (September)、dots的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cbrn", "cybersecurity", "autonomy", "deception" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "8 Alignment", "locator": "https://deploymentsafety.openai.com/gpt-6-astra#section.8", "variantIds": [ "openai-gpt-6-astra-system-card-exploitgym" ], "disposition": "linked", "note": { "en": "Tests include restriction following, deception, realistic work environments and unintended agent communication. Evaluated behavior is not a deployment incident rate.", "zh": "测试涵盖遵守限制、欺骗、真实化工作环境及非预期代理通信；评测结果不等于部署事故率。" }, "reportedProtocols": [ "ExploitGym", "deployment simulation" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "8.7 Verbalized Metagaming and Oversight Gaming", "locator": "https://deploymentsafety.openai.com/gpt-6-astra#subsection.8.7", "variantIds": [], "disposition": "context", "note": { "en": "Card update clarifies that verbalized metagaming is detected in reasoning; oversight-gaming judgments rely on interpretation, not causal identification.", "zh": "卡片更新说明其检测对象为推理中表述的元博弈；监督博弈判断依赖解释而非因果识别。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "9 Monitorability", "locator": "https://deploymentsafety.openai.com/gpt-6-astra#section.9", "variantIds": [], "disposition": "context", "note": { "en": "Separates ordinary and adversarial monitorability. Prompted evasion tests do not establish spontaneous deployed evasion.", "zh": "区分常规与对抗可监测性；诱导规避测试不能证明部署中自发规避。" }, "reportedProtocols": [ "CoT controllability", "monitor evasion", "Sneaky Sandbox Escape" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "10 Preparedness", "locator": "https://deploymentsafety.openai.com/gpt-6-astra#section.10", "variantIds": [ "openai-gpt-6-astra-system-card-exploitbench", "openai-gpt-6-astra-system-card-sre-bench" ], "disposition": "linked", "note": { "en": "Developer assigns Critical cyber and High biological/chemical capability, with safeguards and trusted-access cases; AI self-improvement remains below its High threshold.", "zh": "开发者认定网络能力为Critical、生化能力为High，并配套防护及可信访问；AI自我改进未达其High阈值。" }, "reportedProtocols": [ "ExploitBench", "SRE-Bench", "Sandbox Bench", "KernelGen 1P", "NanoGPT", "PostTrainBench Lite", "MLE-Bench Revised" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "openai-addendum-to-gpt-6-astra-system-card-gpt-6-1-sol", "title": "Addendum to GPT-6 Astra System Card: GPT-6.1 Sol", "kind": "developer", "sourceId": "official-openai-addendum-to-gpt-6-astra-system-card-gpt-6-1-sol", "publicationDate": "2026-09-29", "updatedDate": null, "modelIds": [ "gpt-6-1-sol" ], "publisherIds": [ "openai" ], "summary": { "en": "Developer documentation covering GPT-6.1 Sol.", "zh": "关于GPT-6.1 Sol的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cbrn", "cybersecurity", "autonomy", "deception" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "1 Introduction", "locator": "https://deploymentsafety.openai.com/gpt-6-1-sol#section.1", "variantIds": [], "disposition": "context", "note": { "en": "Addendum evaluates GPT-6.1 Sol and applies Astra safeguards. Prior-model comparison values can reflect later versions rather than launch snapshots.", "zh": "附录评估GPT-6.1 Sol并沿用Astra防护；比较模型数据可能来自后续版本而非发布时快照。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "7 Alignment and 8 Monitorability", "locator": "https://deploymentsafety.openai.com/gpt-6-1-sol#section.7", "variantIds": [ "openai-addendum-to-gpt-6-astra-system-card-gpt-6-1-sol-exploitgym" ], "disposition": "linked", "note": { "en": "Restriction following, deception, realistic deployment simulations and prompted monitor evasion require separate interpretations.", "zh": "遵守限制、欺骗、真实化部署模拟和诱导规避监测须分别解读。" }, "reportedProtocols": [ "ExploitGym", "CoT controllability", "monitor evasion" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "9 Preparedness", "locator": "https://deploymentsafety.openai.com/gpt-6-1-sol#section.9", "variantIds": [ "openai-addendum-to-gpt-6-astra-system-card-gpt-6-1-sol-exploitbench" ], "disposition": "linked", "note": { "en": "Developer treats this model as Critical cyber and High biological/chemical capability; safeguards summary points back to Astra.", "zh": "开发者将其网络能力按Critical、生化能力按High处理；防护摘要指向Astra。" }, "reportedProtocols": [ "ExploitBench", "KernelGen 1P" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "openai-gpt-6-sol-and-gpt-6-luna-october-2026-update", "title": "GPT-6 Sol and GPT-6 Luna: October 2026 update", "kind": "developer", "sourceId": "official-openai-gpt-6-sol-and-gpt-6-luna-october-2026-update", "publicationDate": "2026-10-07", "updatedDate": null, "modelIds": [ "gpt-6-sol-october", "gpt-6-luna-october" ], "publisherIds": [ "openai" ], "summary": { "en": "Developer documentation covering GPT-6 Sol (October), GPT-6 Luna (October).", "zh": "关于GPT-6 Sol (October)、GPT-6 Luna (October)的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cbrn", "cybersecurity", "autonomy", "deception" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "1 Introduction", "locator": "https://deploymentsafety.openai.com/gpt-6-october#section.1", "variantIds": [], "disposition": "context", "note": { "en": "October ChatGPT versions differ from September versions still used in Codex and Work. Preserve the month-specific model identity.", "zh": "10月ChatGPT版本与Codex、Work继续使用的9月版本不同；保留月份级模型身份。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "4 Robustness and 7 Alignment", "locator": "https://deploymentsafety.openai.com/gpt-6-october#section.4", "variantIds": [], "disposition": "context", "note": { "en": "Reports adaptive jailbreak, prompt-injection and restriction-following tests; severity review accompanies regressions.", "zh": "报告自适应越狱、提示注入及遵守限制测试，并对退化结果开展严重性复核。" }, "reportedProtocols": [ "GPT-RED", "Blocker Deception", "Broken Search Tool" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "8 Preparedness", "locator": "https://deploymentsafety.openai.com/gpt-6-october#section.8", "variantIds": [ "openai-gpt-6-sol-and-gpt-6-luna-october-2026-update-exploitbench", "openai-gpt-6-sol-and-gpt-6-luna-october-2026-update-secbench-pro" ], "disposition": "linked", "note": { "en": "October Sol/Luna are treated as High in cyber and biological/chemical domains; capability assessments use maximum reasoning, unlike baseline safety tests.", "zh": "10月Sol/Luna的网络与生化能力均按High处理；能力评估使用最高推理配置，与基础安全测试不同。" }, "reportedProtocols": [ "ExploitBench", "SEC-Bench Pro" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "qwen-qwen3-8-2-4t-a95b-model-card", "title": "Qwen3.8-2.4T-A95B Model Card", "kind": "developer", "sourceId": "official-qwen-qwen3-8-2-4t-a95b-model-card", "publicationDate": null, "updatedDate": null, "modelIds": [ "qwen3-8-2-4t-a95b" ], "publisherIds": [ "qwen" ], "summary": { "en": "Developer documentation covering Qwen3.8-2.4T-A95B.", "zh": "关于Qwen3.8-2.4T-A95B的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Benchmark Results; evaluation footnotes", "locator": "Original source → Benchmark Results; evaluation footnotes", "variantIds": [ "qwen-qwen3-8-2-4t-a95b-model-card-paperbench" ], "disposition": "linked", "note": { "en": "Footnotes document anti-reward-hacking controls and harness differences; no dedicated frontier-risk evaluation found.", "zh": "脚注说明防奖励投机控制与框架差异；未发现独立前沿风险评估。" }, "reportedProtocols": [ "NL2Repo-Bench", "PaperBench" ], "reviewDisposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established" } ] }, { "id": "qwen-qwen3guard-technical-report", "title": "Qwen3Guard Technical Report", "kind": "developer", "sourceId": "official-qwen-qwen3guard-technical-report", "publicationDate": null, "updatedDate": null, "modelIds": [], "publisherIds": [ "qwen" ], "summary": { "en": "Developer documentation covering Qwen3Guard.", "zh": "关于Qwen3Guard的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "2 Safety Policy; benchmark evaluations", "locator": "Original source → 2 Safety Policy; benchmark evaluations", "variantIds": [], "disposition": "context", "note": { "en": "A guardrail-classifier evaluation is governance/safeguard evidence, not an assessment of frontier base-model dangerous capability.", "zh": "防护分类器评估属于治理/防护证据，不能视为前沿基础模型危险能力评估。" }, "reportedProtocols": [ "Qwen3GuardTest", "WildGuardTest" ], "reviewDisposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established" } ] }, { "id": "qwen-qwen3-8-flash-next-technical-report", "title": "On the Design of Qwen3.8-Next Architecture: Evaluation, Efficiency, and Training Stability", "kind": "developer", "sourceId": "official-qwen-qwen3-8-flash-next-technical-report", "publicationDate": "2026-08-26", "updatedDate": null, "modelIds": [ "qwen3-8-flash-next" ], "publisherIds": [ "qwen" ], "summary": { "en": "Developer documentation covering Qwen3.8-Flash-Next.", "zh": "关于Qwen3.8-Flash-Next的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Architecture, evaluation and training-stability sections", "locator": "Original source → Architecture, evaluation and training-stability sections", "variantIds": [], "disposition": "context", "note": { "en": "Technical architecture and optimization report; no dedicated frontier-risk assessment in the reviewed source.", "zh": "技术架构与优化报告；所审来源没有独立前沿风险评估。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established" } ] }, { "id": "xai-grok-4-7-model-card", "title": "Grok 4.7 Model Card", "kind": "developer", "sourceId": "official-xai-grok-4-7-model-card", "publicationDate": "2026-09-21", "updatedDate": "2026-09-21", "modelIds": [ "grok-4-7" ], "publisherIds": [ "xai" ], "summary": { "en": "Developer documentation covering Grok 4.7.", "zh": "关于Grok 4.7的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [ "cbrn", "cybersecurity", "deception" ], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "6 Cyber capabilities and safeguards", "locator": "PDF pp.17–19 (printed 16–18)", "variantIds": [ "xai-grok-4-7-model-card-cybergym", "xai-grok-4-7-model-card-cve-bench" ], "disposition": "linked", "note": { "en": "Capability probes use unrestricted settings; safeguards/refusals are evaluated separately. Peer-card values are imported comparisons, not necessarily matched reruns.", "zh": "能力探测使用无限制配置，防护/拒绝另测；其他模型卡数据不一定为匹配条件复测。" }, "reportedProtocols": [ "CyberGym", "CVE-Bench", "HackerBench v0.3", "CathedralBench" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "7 Biological and chemical capabilities and safeguards", "locator": "PDF pp.20–23 (printed 19–22)", "variantIds": [ "xai-grok-4-7-model-card-wmdp", "xai-grok-4-7-model-card-lab-bench" ], "disposition": "linked", "note": { "en": "Knowledge/capability and refusal are explicitly different signals; only protocol metadata and bounded high-level conclusions retained.", "zh": "知识/能力与拒绝是不同信号；仅保留协议元数据与有边界的高层结论。" }, "reportedProtocols": [ "BioSecBench", "VCT", "Biosecurity VCT", "BioUseBench", "WMDP", "LAB-Bench", "ProtocolQA", "BixBench" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "8 Jailbreaks and robustness", "locator": "PDF p.23 (printed 22)", "variantIds": [ "xai-grok-4-7-model-card-strongreject" ], "disposition": "linked", "note": { "en": "Production safeguards are stress-tested under single-turn, multiturn and longer attacks; residual failures motivate monitoring.", "zh": "生产防护接受单轮、多轮与长程攻击测试；剩余失败仍需监测。" }, "reportedProtocols": [ "StrongREJECT", "Crescendo" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" }, { "heading": "11 Behaviors", "locator": "PDF pp.26–27 (printed 25–26)", "variantIds": [], "disposition": "context", "note": { "en": "MASK-Rectified changes grading relative to MASK, so its honesty result cannot be pooled with ordinary MASK.", "zh": "MASK-Rectified改变MASK评分方式，诚实性结果不能与普通MASK合并。" }, "reportedProtocols": [ "MASK-Rectified" ], "reviewDisposition": "source-section-reviewed; numerical-comparison-not-assessed" } ] }, { "id": "zai-glm-5-2-model-card", "title": "GLM-5.2 Model Card", "kind": "developer", "sourceId": "official-zai-glm-5-2-model-card", "publicationDate": null, "updatedDate": null, "modelIds": [ "glm-5-2" ], "publisherIds": [ "zai" ], "summary": { "en": "Developer documentation covering GLM-5.2.", "zh": "关于GLM-5.2的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Benchmark; Footnote", "locator": "Original source → Benchmark; Footnote", "variantIds": [ "zai-glm-5-2-model-card-swe-bench-pro" ], "disposition": "linked", "note": { "en": "General capability comparisons and harness notes; no dedicated frontier-risk evaluation or safeguard-effectiveness section located in this card.", "zh": "一般能力比较与测试框架说明；本卡未发现独立前沿风险评估或防护有效性章节。" }, "reportedProtocols": [ "SWE-bench Pro", "DeepSWE", "MCP-Atlas" ], "reviewDisposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established" } ] }, { "id": "zai-glm-5-3-bf16-model-card", "title": "GLM-5.3-BF16 Model Card", "kind": "developer", "sourceId": "official-zai-glm-5-3-bf16-model-card", "publicationDate": null, "updatedDate": null, "modelIds": [ "glm-5-3-bf16" ], "publisherIds": [ "zai" ], "summary": { "en": "Developer documentation covering GLM-5.3-BF16.", "zh": "关于GLM-5.3-BF16的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "Benchmark; Footnotes", "locator": "Original source → Benchmark; Footnotes", "variantIds": [], "disposition": "context", "note": { "en": "Latest linked card references the GLM-5 report; it does not independently establish the safety findings of GLM-5.2 external evaluations.", "zh": "最新链接卡引用GLM-5报告；不能据此建立外部GLM-5.2评估的安全结论。" }, "reportedProtocols": [ "DeepSWE" ], "reviewDisposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established" } ] }, { "id": "zai-glm-5-from-vibe-coding-to-agentic-engineering", "title": "GLM-5: from Vibe Coding to Agentic Engineering", "kind": "developer", "sourceId": "official-zai-glm-5-from-vibe-coding-to-agentic-engineering", "publicationDate": null, "updatedDate": null, "modelIds": [ "glm-5" ], "publisherIds": [ "zai" ], "summary": { "en": "Developer documentation covering GLM-5.", "zh": "关于GLM-5的开发者文档，核查范围和已关联协议见下方记录。" }, "areas": [], "reviewStatus": "bounded-sections", "scopeNote": { "en": "Developer-authored disclosure. Selected source sections were read; numerical results and comparability have not been audited. NUWA has not independently replicated these experiments.", "zh": "开发者自行发布的披露。已阅读所列部分章节，未核验数值结果和比较资格。NUWA 未独立复现这些实验。" }, "sections": [ { "heading": "3.4 General RL; agentic training", "locator": "Original source → 3.4 General RL; agentic training", "variantIds": [], "disposition": "context", "note": { "en": "Reward-model robustness and training-time reward hacking are discussed; there is no dedicated catastrophic-risk assessment in the reviewed report.", "zh": "讨论奖励模型稳健性与训练奖励投机；所审报告没有独立灾难风险评估。" }, "reportedProtocols": [], "reviewDisposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established" } ] } ], "protocolVariants": [ { "id": "neo-kimi-k3-charxiv", "protocolId": "charxiv", "name": "CharXiv with images withheld", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§1.1.1; pp. 7–9", "primaryDimension": "behavior", "measurementTarget": { "en": "Chart questions with required images withheld; attempts split into correct and incorrect.", "zh": "移除必要图像的图表问题；回答尝试区分正确与错误。" }, "taskScope": { "en": "Chart questions with required images withheld; attempts split into correct and incorrect.", "zh": "移除必要图像的图表问题；回答尝试区分正确与错误。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "The original visual capability benchmark and the missing-image behavioral test measure different constructs.", "zh": "原始视觉能力基准与缺失图像行为测试测量不同构念。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "The modified task withholds required inputs and observes unsupported attempts to answer; it is not the original capability test.", "zh": "改造任务移除了必要输入，观察缺乏依据的作答尝试，不是原始能力测验。" }, "locator": "§1.1.1; pp. 7–9" } ] }, { "id": "neo-kimi-k3-lab-bench", "protocolId": "lab-bench", "name": "LAB-Bench with figures/tables withheld", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§1.1.2; pp. 8–10", "primaryDimension": "behavior", "measurementTarget": { "en": "FigQA and TableQA with required figures or tables withheld, retaining the insufficient-information answer option.", "zh": "移除 FigQA 和 TableQA 必要图表，保留信息不足选项。" }, "taskScope": { "en": "FigQA and TableQA with required figures or tables withheld, retaining the insufficient-information answer option.", "zh": "移除 FigQA 和 TableQA 必要图表，保留信息不足选项。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "These are general scientific capabilities, not end-to-end misuse tasks or direct harm probabilities.", "zh": "衡量一般科研能力，不是端到端滥用任务或直接伤害概率。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "The modified task withholds required inputs and observes unsupported attempts to answer; it is not the original capability test.", "zh": "改造任务移除了必要输入，观察缺乏依据的作答尝试，不是原始能力测验。" }, "locator": "§1.1.2; pp. 8–10" } ] }, { "id": "neo-kimi-k3-abstentionbench", "protocolId": "abstentionbench", "name": "AbstentionBench", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§1.2; p. 11", "primaryDimension": "behavior", "measurementTarget": { "en": "Tests abstention across 20 datasets with unknown, underspecified, false-premise, subjective or outdated questions.", "zh": "在 20 个数据集中检验未知答案、信息不足、错误前提、主观或过时问题的弃答。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Abstention is distinct from a safety refusal and does not alone establish intent or downstream harm.", "zh": "弃答不同于安全拒答，不能单独证明意图或下游伤害。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-kimi-k3-preseeded-audits", "protocolId": null, "name": "Pre-seeded alignment audits", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§1.3; pp. 12–18", "primaryDimension": "behavior", "measurementTarget": { "en": "Pre-seeded alignment audits in the report-defined setting.", "zh": "报告定义条件中的 Pre-seeded alignment audits。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-kimi-k3-cybench", "protocolId": "cybench", "name": "Cybench", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§2.1.1; p. 19", "primaryDimension": "capability", "measurementTarget": { "en": "Task completion on 40 professional CTF problems.", "zh": "40 项专业 CTF 任务的完成情况。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Competition tasks and supplied guidance differ from operational attacks.", "zh": "竞赛任务与提供的提示不同于现实攻击条件。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "§2.1.1; p. 19" } ] }, { "id": "neo-kimi-k3-evmbench", "protocolId": "evmbench", "name": "EVMbench", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§2.1.2; p. 19", "primaryDimension": "capability", "measurementTarget": { "en": "Separates detection, patching and exploitation modes with their stated task sets.", "zh": "分别组织检测、修补和利用模式及对应任务集。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Modes and task subsets cannot be combined into an undifferentiated cyber-risk score.", "zh": "不同模式和子集不能合并成无区分的网络风险分数。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "direct", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§2.1.2; p. 19" } ] }, { "id": "neo-kimi-k3-cybergym", "protocolId": "cybergym", "name": "CyberGym infrastructure probing", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§2.1.3–2.1.4; pp. 20–23", "primaryDimension": "behavior", "measurementTarget": { "en": "Transcript analysis of benchmark recognition, grader reasoning and withheld-endpoint probing; not task-success scoring.", "zh": "分析基准识别、判分器推理和隐藏端点探测轨迹，不是任务成功评分。" }, "taskScope": { "en": "Transcript analysis of benchmark recognition, grader reasoning and withheld-endpoint probing; not task-success scoring.", "zh": "分析基准识别、判分器推理和隐藏端点探测轨迹，不是任务成功评分。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Vulnerability reproduction is narrower than autonomous compromise of deployed systems.", "zh": "漏洞复现的范围小于对部署系统的自主入侵。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "evaluation-gaming", "relation": "context", "rationale": { "en": "The report observes benchmark-infrastructure awareness and probing, rather than task-solving capability.", "zh": "报告观察基准设施识别和探测，而非任务求解能力。" }, "locator": "§2.1.3–2.1.4; pp. 20–23" } ] }, { "id": "neo-kimi-k3-theagentcompany", "protocolId": "theagentcompany", "name": "TheAgentCompany boundary behavior", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§2.2; pp. 24–25", "primaryDimension": "capability", "measurementTarget": { "en": "Combines browsing, coding, file operations and interaction with simulated coworkers.", "zh": "结合浏览、编程、文件操作和与模拟同事的交互。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Task success and evaluation-boundary behavior are separate outcomes; a broken environment is not an ordinary task failure.", "zh": "任务成功与评测边界行为属于不同结果；环境故障不是普通任务失败。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "sustained-autonomous-work", "relation": "enabling", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§2.2; pp. 24–25" } ] }, { "id": "neo-kimi-k3-impossiblebench", "protocolId": "impossiblebench", "name": "ImpossibleBench", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§2.3; p. 26", "primaryDimension": "propensity", "measurementTarget": { "en": "Constructs impossible variants of coding tasks and examines specification-violating passes.", "zh": "构造不可同时满足要求的编程任务变体，检验违反规范的通过结果。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "The cheating rate is conditional on contradictions, test access and scaffolding; it is not a general dishonesty rate.", "zh": "作弊率受矛盾条件、测试访问和脚手架限制，不是一般不诚实率。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "evaluation-gaming", "relation": "direct", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§2.3; p. 26" } ] }, { "id": "neo-kimi-k3-apollo-scenario-awareness", "protocolId": null, "name": "Apollo-scenario recognition", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§2.4.1; p. 28", "primaryDimension": "behavior", "measurementTarget": { "en": "Apollo-scenario recognition in the report-defined setting.", "zh": "报告定义条件中的 Apollo-scenario recognition。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-kimi-k3-evalawarebench", "protocolId": "evalawarebench", "name": "EvalAwareBench", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§2.4.2; pp. 28–30", "primaryDimension": "behavior", "measurementTarget": { "en": "Varies evaluation cues while tracking verbalized evaluation awareness in controlled tasks.", "zh": "在受控任务中改变评测线索，记录显式表达的评测感知。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Unexpressed awareness is not measured; recognition of familiar benchmarks can differ from general cue response.", "zh": "不能测量未表达的感知；熟悉基准的识别可能不同于一般线索反应。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "evaluation-gaming", "relation": "context", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§2.4.2; pp. 28–30" } ] }, { "id": "neo-kimi-k3-alignment-faking", "protocolId": "alignment-faking", "name": "Alignment faking", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§3.1; p. 32", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-dependent response differences and reported traces.", "zh": "不同试验条件下的响应差异及所报告轨迹。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "The setting explicitly gives fictional training information; observations do not establish an enduring hidden goal in deployment.", "zh": "试验明确提供虚构训练信息；观察结果不证明部署中存在持久隐藏目标。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "Repeated condition contrasts probe a bounded tendency to present compliant responses while preserving a conflicting objective.", "zh": "重复条件对照测试在目标冲突时表现顺从响应的有界倾向。" }, "locator": "§3.1; p. 32" } ] }, { "id": "neo-kimi-k3-agentic-misalignment", "protocolId": "agentic-misalignment", "name": "Agentic Misalignment, historical", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§3.2.1; pp. 34–36", "primaryDimension": "propensity", "measurementTarget": { "en": "Conditional rates and observed actions in controlled simulations.", "zh": "受控模拟中的条件性发生率与可观察行动。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "No real people or systems were harmed; the study does not estimate prevalence in deployment.", "zh": "试验未对真实人员或系统造成损害，也不估计部署环境中的发生率。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "unsanctioned-action", "relation": "direct", "rationale": { "en": "Varied controlled scenarios test actions contrary to the assigned benign role.", "zh": "变化的受控情境测试与既定良性职责相违背的行动。" }, "locator": "§3.2.1; pp. 34–36" }, { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "Some scenarios test concealed harmful actions in pursuit of an assigned goal.", "zh": "部分情境测试在追求指定目标时隐蔽实施有害行动。" }, "locator": "§3.2.1; pp. 34–36" } ] }, { "id": "neo-kimi-k3-translated-misalignment", "protocolId": null, "name": "Agentic Misalignment, translated", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§3.2.2; pp. 37–39", "primaryDimension": "behavior", "measurementTarget": { "en": "Agentic Misalignment, translated in the report-defined setting.", "zh": "报告定义条件中的 Agentic Misalignment, translated。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-kimi-k3-palisade-shutdown", "protocolId": "palisade-shutdown", "name": "Shutdown resistance", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§4.1; pp. 40–41", "primaryDimension": "propensity", "measurementTarget": { "en": "Conditional frequency of shutdown-script interference and observed action traces.", "zh": "关闭脚本受干预的条件性频率及行动轨迹。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "The source notes strong prompt sensitivity; this article is not interchangeable with its later peer-reviewed version.", "zh": "来源指出结果对提示敏感；该文章不可与之后的同行评审版本混用。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "shutdown-resistance", "relation": "direct", "rationale": { "en": "Repeated trials support a condition-bounded tendency estimate, not a stable survival motive.", "zh": "重复试验支持有条件的倾向估计，不证明稳定的自保动机。" }, "locator": "§4.1; pp. 40–41" }, { "area": "autonomy", "questionId": "shutdown-resistance", "relation": "direct", "rationale": { "en": "The source reports concrete modifications to the shutdown script in individual runs.", "zh": "来源报告了具体试验中对关闭脚本的修改。" }, "locator": "§4.1; pp. 40–41" } ] }, { "id": "neo-kimi-k3-runpod-continuity", "protocolId": null, "name": "RunPod service continuity", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§4.2; pp. 40–43", "primaryDimension": "behavior", "measurementTarget": { "en": "RunPod service continuity in the report-defined setting.", "zh": "报告定义条件中的 RunPod service continuity。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-kimi-k3-petri", "protocolId": "petri", "name": "Petri", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§4.3; pp. 43–44", "primaryDimension": "behavior", "measurementTarget": { "en": "An auditor explores seeded multi-turn interactions, with rubric-based judging of target transcripts.", "zh": "审计者探索以种子设定启动的多轮交互，并按量表判读目标轨迹。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Low average audit scores do not rule out rare severe cases; results depend on seeds, auditor, judge and target configuration.", "zh": "较低平均审计分数不排除罕见严重案例；结果依赖种子、审计者、判别器和目标配置。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-kimi-k3-truthful-sad", "protocolId": "truthful-sad", "name": "Situational Awareness Dataset", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§5.1; p. 45", "primaryDimension": "capability", "measurementTarget": { "en": "Task accuracy on whether the model recognizes itself, its context, and possible influence.", "zh": "模型识别自身、所处环境及可影响范围的任务准确率。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Situational awareness is a possible prerequisite for evaluation gaming, not evidence that gaming occurred.", "zh": "情境意识可能是评测规避的前提，不证明规避行为已经发生。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "evaluation-gaming", "relation": "enabling", "rationale": { "en": "The benchmark measures awareness abilities; it does not test deliberate sandbagging.", "zh": "该基准测量识别能力，不测试有意低表现。" }, "locator": "§5.1; p. 45" } ] }, { "id": "neo-kimi-k3-stealth-evals", "protocolId": "stealth-evals", "name": "GDM Stealth evaluations", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§5.2; p. 46", "primaryDimension": "capability", "measurementTarget": { "en": "Reasoning about oversight and deployment context.", "zh": "有关监督机制与部署情境的推理。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Prerequisite capability is distinct from a tendency to scheme.", "zh": "前置能力不同于实施隐蔽谋划的倾向。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "evaluation-gaming", "relation": "enabling", "rationale": { "en": "The source measures a prerequisite capability; it does not show the risk behavior.", "zh": "来源测量前提能力，并未观察到对应风险行为。" }, "locator": "§5.2; p. 46" } ] }, { "id": "neo-kimi-k3-hint-monitorability", "protocolId": null, "name": "Hint-influence monitorability", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§5.3.1; p. 48", "primaryDimension": "behavior", "measurementTarget": { "en": "Hint-influence monitorability in the report-defined setting.", "zh": "报告定义条件中的 Hint-influence monitorability。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-kimi-k3-cot-controllability", "protocolId": null, "name": "CoT controllability", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§5.3.2; p. 48", "primaryDimension": "behavior", "measurementTarget": { "en": "CoT controllability in the report-defined setting.", "zh": "报告定义条件中的 CoT controllability。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-kimi-k3-shade-arena", "protocolId": "shade-arena", "name": "SHADE-Arena", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§5.3.3; p. 49", "primaryDimension": "capability", "measurementTarget": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "An assigned malicious side task measures ability to execute it, not spontaneous desire to sabotage.", "zh": "被指派的恶意副任务测量执行能力，不证明自发破坏倾向。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "The protocol tests whether agents can complete a covert objective while doing legitimate work.", "zh": "协议测试智能体能否在执行正当任务时完成隐蔽目标。" }, "locator": "§5.3.3; p. 49" } ] }, { "id": "neo-kimi-k3-agentharm", "protocolId": "agentharm", "name": "AgentHarm", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§6.1; p. 51", "primaryDimension": "behavior", "measurementTarget": { "en": "Harmful task completion and refusal under specified attacks.", "zh": "指定攻击下的有害任务完成和拒答表现。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Synthetic tools and task completion do not directly measure real-world harm.", "zh": "合成工具与任务完成情况不直接衡量现实损害。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-kimi-k3-harmful-finetuning", "protocolId": null, "name": "Harmful fine-tuning", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§6.2; pp. 52–53", "primaryDimension": "behavior", "measurementTarget": { "en": "Harmful fine-tuning in the report-defined setting.", "zh": "报告定义条件中的 Harmful fine-tuning。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-kimi-k3-fortress", "protocolId": "fortress", "name": "FORTRESS", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§6.3; pp. 54–57", "primaryDimension": "behavior", "measurementTarget": { "en": "Evaluates safeguards on national-security and public-safety tasks, separating hazardous requests from benign-use utility.", "zh": "在国家安全与公共安全任务上评测防护，区分高危请求与正常用途的有用性。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Judge reliability, task definitions and over-refusal affect the meaning of the reported scores.", "zh": "评判器可靠性、任务定义和过度拒答影响所报告分数的含义。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-kimi-k3-ape", "protocolId": "ape", "name": "APE", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§7.1; p. 58", "primaryDimension": "propensity", "measurementTarget": { "en": "Frequency of persuasive attempts under specified conversational conditions.", "zh": "特定对话条件下说服尝试的发生率。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Simulated attempts are not measured human belief change or large-scale disinformation reach.", "zh": "模拟尝试不等于已测量的人类信念变化或大规模虚假信息传播。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "disinformation", "questionId": "persuasion-attempts", "relation": "direct", "rationale": { "en": "The study explicitly varies conditions and scores willingness to attempt persuasion.", "zh": "研究明确变化条件并评估尝试说服的意愿。" }, "locator": "§7.1; p. 58" } ] }, { "id": "neo-kimi-k3-persusafety", "protocolId": "persusafety", "name": "PersuSafety with adapted judge", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§7.2; pp. 59–60", "primaryDimension": "behavior", "measurementTarget": { "en": "Defines persuasion scenarios, simulated conversations and safety assessment; adapted judges are separate variants.", "zh": "定义说服情境、模拟对话与安全评测；改造判别器作为独立变体。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Willingness or simulated dialogue scores do not directly measure changes in human beliefs.", "zh": "参与意愿或模拟对话评分不直接测量人的信念变化。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "disinformation", "questionId": "persuasion-attempts", "relation": "direct", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§7.2; pp. 59–60" } ] }, { "id": "neo-kimi-k3-psychosisbench", "protocolId": null, "name": "PsychosisBench", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§7.3; p. 61", "primaryDimension": "behavior", "measurementTarget": { "en": "PsychosisBench in the report-defined setting.", "zh": "报告定义条件中的 PsychosisBench。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-kimi-k3-deceptionbench", "protocolId": "deceptionbench", "name": "DeceptionBench", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§7.4; pp. 62–63", "primaryDimension": "propensity", "measurementTarget": { "en": "Evaluates 150 authored scenarios across five social domains, including neutral, incentive and pressure conditions.", "zh": "评测五个社会领域的 150 个编写情境，区分中性、激励与压力条件。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Authored scenarios and model-based judging limit generalization; elicited deception does not establish covert long-term scheming.", "zh": "编写情境和模型判别限制外推；诱发的欺骗不证明隐蔽长程谋划。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§7.4; pp. 62–63" } ] }, { "id": "neo-kimi-k3-mask", "protocolId": "mask", "name": "MASK", "version": "September 2026", "sourceId": "neo-kimi-k3-source", "locator": "§7.5; pp. 64–65", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "§7.5; pp. 64–65" } ] }, { "id": "neo-glm52-mask", "protocolId": "mask", "name": "MASK", "version": "3 July 2026", "sourceId": "neo-glm52-source", "locator": "§1; Figure 1", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "§1; Figure 1" } ] }, { "id": "neo-glm52-deceptionbench", "protocolId": "deceptionbench", "name": "DeceptionBench", "version": "3 July 2026", "sourceId": "neo-glm52-source", "locator": "§1; Figure 2", "primaryDimension": "propensity", "measurementTarget": { "en": "Evaluates 150 authored scenarios across five social domains, including neutral, incentive and pressure conditions.", "zh": "评测五个社会领域的 150 个编写情境，区分中性、激励与压力条件。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Authored scenarios and model-based judging limit generalization; elicited deception does not establish covert long-term scheming.", "zh": "编写情境和模型判别限制外推；诱发的欺骗不证明隐蔽长程谋划。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§1; Figure 2" } ] }, { "id": "neo-glm52-gram", "protocolId": null, "name": "Gram automated audits", "version": "3 July 2026", "sourceId": "neo-glm52-source", "locator": "§1; Figures 3–4", "primaryDimension": "behavior", "measurementTarget": { "en": "Gram automated audits in the report-defined setting.", "zh": "报告定义条件中的 Gram automated audits。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-glm52-ape", "protocolId": "ape", "name": "APE", "version": "3 July 2026", "sourceId": "neo-glm52-source", "locator": "§2; Figure 5", "primaryDimension": "propensity", "measurementTarget": { "en": "Frequency of persuasive attempts under specified conversational conditions.", "zh": "特定对话条件下说服尝试的发生率。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Simulated attempts are not measured human belief change or large-scale disinformation reach.", "zh": "模拟尝试不等于已测量的人类信念变化或大规模虚假信息传播。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "disinformation", "questionId": "persuasion-attempts", "relation": "direct", "rationale": { "en": "The study explicitly varies conditions and scores willingness to attempt persuasion.", "zh": "研究明确变化条件并评估尝试说服的意愿。" }, "locator": "§2; Figure 5" } ] }, { "id": "neo-glm52-persusafety", "protocolId": "persusafety", "name": "PersuSafety adapted free-generation judge", "version": "3 July 2026", "sourceId": "neo-glm52-source", "locator": "§2; Figure 6", "primaryDimension": "behavior", "measurementTarget": { "en": "Free-form generation with an APE-style StrongREJECT judge, replacing the original explicit willingness question.", "zh": "自由生成并采用 APE 风格 StrongREJECT 判别器，替代原始显式参与意愿提问。" }, "taskScope": { "en": "Free-form generation with an APE-style StrongREJECT judge, replacing the original explicit willingness question.", "zh": "自由生成并采用 APE 风格 StrongREJECT 判别器，替代原始显式参与意愿提问。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Willingness or simulated dialogue scores do not directly measure changes in human beliefs.", "zh": "参与意愿或模拟对话评分不直接测量人的信念变化。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "disinformation", "questionId": "persuasion-attempts", "relation": "direct", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§2; Figure 6" } ] }, { "id": "neo-glm52-agentic-misalignment", "protocolId": "agentic-misalignment", "name": "Agentic Misalignment awareness trend", "version": "3 July 2026", "sourceId": "neo-glm52-source", "locator": "§3; Figure 7", "primaryDimension": "propensity", "measurementTarget": { "en": "Conditional rates and observed actions in controlled simulations.", "zh": "受控模拟中的条件性发生率与可观察行动。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "No real people or systems were harmed; the study does not estimate prevalence in deployment.", "zh": "试验未对真实人员或系统造成损害，也不估计部署环境中的发生率。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "unsanctioned-action", "relation": "direct", "rationale": { "en": "Varied controlled scenarios test actions contrary to the assigned benign role.", "zh": "变化的受控情境测试与既定良性职责相违背的行动。" }, "locator": "§3; Figure 7" }, { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "Some scenarios test concealed harmful actions in pursuit of an assigned goal.", "zh": "部分情境测试在追求指定目标时隐蔽实施有害行动。" }, "locator": "§3; Figure 7" } ] }, { "id": "neo-glm52-mask-awareness", "protocolId": null, "name": "MASK verbalized awareness", "version": "3 July 2026", "sourceId": "neo-glm52-source", "locator": "§3", "primaryDimension": "behavior", "measurementTarget": { "en": "MASK verbalized awareness in the report-defined setting.", "zh": "报告定义条件中的 MASK verbalized awareness。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-deepseek-v4-cybench", "protocolId": "cybench", "name": "Cybench", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§1.1", "primaryDimension": "capability", "measurementTarget": { "en": "Task completion on 40 professional CTF problems.", "zh": "40 项专业 CTF 任务的完成情况。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Competition tasks and supplied guidance differ from operational attacks.", "zh": "竞赛任务与提供的提示不同于现实攻击条件。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "§1.1" } ] }, { "id": "neo-deepseek-v4-evmbench", "protocolId": "evmbench", "name": "EVMbench", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§1.2", "primaryDimension": "capability", "measurementTarget": { "en": "Separates detection, patching and exploitation modes with their stated task sets.", "zh": "分别组织检测、修补和利用模式及对应任务集。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Modes and task subsets cannot be combined into an undifferentiated cyber-risk score.", "zh": "不同模式和子集不能合并成无区分的网络风险分数。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "direct", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§1.2" } ] }, { "id": "neo-deepseek-v4-wmdp", "protocolId": "wmdp", "name": "WMDP", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§2.1", "primaryDimension": "capability", "measurementTarget": { "en": "Multiple-choice knowledge measurements.", "zh": "选择题形式的知识测量。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Knowledge is a proxy; scores do not establish practical harmful-task ability.", "zh": "知识属于代理指标；分数不能确定实际有害任务能力。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "cbrn", "questionId": "hazardous-knowledge", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "§2.1" } ] }, { "id": "neo-deepseek-v4-lab-bench", "protocolId": "lab-bench", "name": "LAB-Bench", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§2.2", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates a range of biology research skills with structured questions; text-only or missing-input subsets remain separate protocols.", "zh": "用结构化问题评测生物研究技能；纯文本或缺失输入子集采用独立协议。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "These are general scientific capabilities, not end-to-end misuse tasks or direct harm probabilities.", "zh": "衡量一般科研能力，不是端到端滥用任务或直接伤害概率。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "cbrn", "questionId": "task-uplift", "relation": "enabling", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§2.2" } ] }, { "id": "neo-deepseek-v4-mask", "protocolId": "mask", "name": "MASK", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§3.1", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "§3.1" } ] }, { "id": "neo-deepseek-v4-deceptionbench", "protocolId": "deceptionbench", "name": "DeceptionBench", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§3.2", "primaryDimension": "propensity", "measurementTarget": { "en": "Evaluates 150 authored scenarios across five social domains, including neutral, incentive and pressure conditions.", "zh": "评测五个社会领域的 150 个编写情境，区分中性、激励与压力条件。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Authored scenarios and model-based judging limit generalization; elicited deception does not establish covert long-term scheming.", "zh": "编写情境和模型判别限制外推；诱发的欺骗不证明隐蔽长程谋划。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§3.2" } ] }, { "id": "neo-deepseek-v4-ape", "protocolId": "ape", "name": "APE", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§3.3", "primaryDimension": "propensity", "measurementTarget": { "en": "Frequency of persuasive attempts under specified conversational conditions.", "zh": "特定对话条件下说服尝试的发生率。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Simulated attempts are not measured human belief change or large-scale disinformation reach.", "zh": "模拟尝试不等于已测量的人类信念变化或大规模虚假信息传播。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "disinformation", "questionId": "persuasion-attempts", "relation": "direct", "rationale": { "en": "The study explicitly varies conditions and scores willingness to attempt persuasion.", "zh": "研究明确变化条件并评估尝试说服的意愿。" }, "locator": "§3.3" } ] }, { "id": "neo-deepseek-v4-persusafety", "protocolId": "persusafety", "name": "PersuSafety adapted judge", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§3.4", "primaryDimension": "behavior", "measurementTarget": { "en": "Defines persuasion scenarios, simulated conversations and safety assessment; adapted judges are separate variants.", "zh": "定义说服情境、模拟对话与安全评测；改造判别器作为独立变体。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Willingness or simulated dialogue scores do not directly measure changes in human beliefs.", "zh": "参与意愿或模拟对话评分不直接测量人的信念变化。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "disinformation", "questionId": "persuasion-attempts", "relation": "direct", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§3.4" } ] }, { "id": "neo-deepseek-v4-alignment-faking", "protocolId": "alignment-faking", "name": "Alignment faking", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§4.1", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-dependent response differences and reported traces.", "zh": "不同试验条件下的响应差异及所报告轨迹。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "The setting explicitly gives fictional training information; observations do not establish an enduring hidden goal in deployment.", "zh": "试验明确提供虚构训练信息；观察结果不证明部署中存在持久隐藏目标。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "Repeated condition contrasts probe a bounded tendency to present compliant responses while preserving a conflicting objective.", "zh": "重复条件对照测试在目标冲突时表现顺从响应的有界倾向。" }, "locator": "§4.1" } ] }, { "id": "neo-deepseek-v4-agentic-misalignment", "protocolId": "agentic-misalignment", "name": "Agentic Misalignment", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§4.2", "primaryDimension": "propensity", "measurementTarget": { "en": "Conditional rates and observed actions in controlled simulations.", "zh": "受控模拟中的条件性发生率与可观察行动。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "No real people or systems were harmed; the study does not estimate prevalence in deployment.", "zh": "试验未对真实人员或系统造成损害，也不估计部署环境中的发生率。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "unsanctioned-action", "relation": "direct", "rationale": { "en": "Varied controlled scenarios test actions contrary to the assigned benign role.", "zh": "变化的受控情境测试与既定良性职责相违背的行动。" }, "locator": "§4.2" }, { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "Some scenarios test concealed harmful actions in pursuit of an assigned goal.", "zh": "部分情境测试在追求指定目标时隐蔽实施有害行动。" }, "locator": "§4.2" } ] }, { "id": "neo-deepseek-v4-palisade-shutdown", "protocolId": "palisade-shutdown", "name": "Shutdown resistance", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§4.3", "primaryDimension": "propensity", "measurementTarget": { "en": "Conditional frequency of shutdown-script interference and observed action traces.", "zh": "关闭脚本受干预的条件性频率及行动轨迹。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "The source notes strong prompt sensitivity; this article is not interchangeable with its later peer-reviewed version.", "zh": "来源指出结果对提示敏感；该文章不可与之后的同行评审版本混用。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "shutdown-resistance", "relation": "direct", "rationale": { "en": "Repeated trials support a condition-bounded tendency estimate, not a stable survival motive.", "zh": "重复试验支持有条件的倾向估计，不证明稳定的自保动机。" }, "locator": "§4.3" }, { "area": "autonomy", "questionId": "shutdown-resistance", "relation": "direct", "rationale": { "en": "The source reports concrete modifications to the shutdown script in individual runs.", "zh": "来源报告了具体试验中对关闭脚本的修改。" }, "locator": "§4.3" } ] }, { "id": "neo-deepseek-v4-impossiblebench", "protocolId": "impossiblebench", "name": "Reward hacking", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§4", "primaryDimension": "propensity", "measurementTarget": { "en": "Constructs impossible variants of coding tasks and examines specification-violating passes.", "zh": "构造不可同时满足要求的编程任务变体，检验违反规范的通过结果。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "The cheating rate is conditional on contradictions, test access and scaffolding; it is not a general dishonesty rate.", "zh": "作弊率受矛盾条件、测试访问和脚手架限制，不是一般不诚实率。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "evaluation-gaming", "relation": "direct", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§4" } ] }, { "id": "neo-deepseek-v4-fortress", "protocolId": "fortress", "name": "Adversarial robustness", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§5", "primaryDimension": "behavior", "measurementTarget": { "en": "Evaluates safeguards on national-security and public-safety tasks, separating hazardous requests from benign-use utility.", "zh": "在国家安全与公共安全任务上评测防护，区分高危请求与正常用途的有用性。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Judge reliability, task definitions and over-refusal affect the meaning of the reported scores.", "zh": "评判器可靠性、任务定义和过度拒答影响所报告分数的含义。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-deepseek-v4-evaluation-awareness", "protocolId": null, "name": "Evaluation awareness", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§6", "primaryDimension": "behavior", "measurementTarget": { "en": "Evaluation awareness in the report-defined setting.", "zh": "报告定义条件中的 Evaluation awareness。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-deepseek-v4-petri", "protocolId": "petri", "name": "Automated alignment audit", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§6", "primaryDimension": "behavior", "measurementTarget": { "en": "An auditor explores seeded multi-turn interactions, with rubric-based judging of target transcripts.", "zh": "审计者探索以种子设定启动的多轮交互，并按量表判读目标轨迹。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Low average audit scores do not rule out rare severe cases; results depend on seeds, auditor, judge and target configuration.", "zh": "较低平均审计分数不排除罕见严重案例；结果依赖种子、审计者、判别器和目标配置。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "neo-deepseek-v4-judge-sensitivity", "protocolId": null, "name": "Safety-judge sensitivity", "version": "v1.1 · 29 May 2026", "sourceId": "neo-deepseek-v4-source", "locator": "§7", "primaryDimension": "behavior", "measurementTarget": { "en": "Safety-judge sensitivity in the report-defined setting.", "zh": "报告定义条件中的 Safety-judge sensitivity。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "saferai-glm52-cybench", "protocolId": "cybench", "name": "Cybench", "version": "v1.1 · revised 26 August 2026", "sourceId": "saferai-glm52-source", "locator": "§2.1; Appendix A.1", "primaryDimension": "capability", "measurementTarget": { "en": "Task completion on 40 professional CTF problems.", "zh": "40 项专业 CTF 任务的完成情况。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Competition tasks and supplied guidance differ from operational attacks.", "zh": "竞赛任务与提供的提示不同于现实攻击条件。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "§2.1; Appendix A.1" } ] }, { "id": "saferai-glm52-cybergym", "protocolId": "cybergym", "name": "CyberGym hard subset with token budgets", "version": "v1.1 · revised 26 August 2026", "sourceId": "saferai-glm52-source", "locator": "§2.2; Appendix A.2", "primaryDimension": "capability", "measurementTarget": { "en": "Lyptus hard-task subset, distinct 2M, 10M and 50M token budgets; baseline data reused from Lyptus.", "zh": "Lyptus 高难度子集，分别采用 2M、10M 和 50M Token 预算；基线数据引用 Lyptus。" }, "taskScope": { "en": "Lyptus hard-task subset, distinct 2M, 10M and 50M token budgets; baseline data reused from Lyptus.", "zh": "Lyptus 高难度子集，分别采用 2M、10M 和 50M Token 预算；基线数据引用 Lyptus。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Vulnerability reproduction is narrower than autonomous compromise of deployed systems.", "zh": "漏洞复现的范围小于对部署系统的自主入侵。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "vulnerability-reproduction", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "§2.2; Appendix A.2" } ] }, { "id": "saferai-glm52-lab-bench", "protocolId": "lab-bench", "name": "LAB-Bench text subtasks", "version": "v1.1 · revised 26 August 2026", "sourceId": "saferai-glm52-source", "locator": "§3.1; Appendix A.3", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates a range of biology research skills with structured questions; text-only or missing-input subsets remain separate protocols.", "zh": "用结构化问题评测生物研究技能；纯文本或缺失输入子集采用独立协议。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "These are general scientific capabilities, not end-to-end misuse tasks or direct harm probabilities.", "zh": "衡量一般科研能力，不是端到端滥用任务或直接伤害概率。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "cbrn", "questionId": "task-uplift", "relation": "enabling", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§3.1; Appendix A.3" } ] }, { "id": "saferai-glm52-biomysterybench", "protocolId": "biomysterybench", "name": "BioMysteryBench public release", "version": "v1.1 · revised 26 August 2026", "sourceId": "saferai-glm52-source", "locator": "§3.2; Appendix A.4", "primaryDimension": "capability", "measurementTarget": { "en": "Assesses scientific analysis over anonymized datasets; versions, human reference subsets and attempt budgets are kept explicit.", "zh": "评测匿名数据集上的科研分析；明确版本、人类参照子集和尝试预算。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Aggregate scientific performance does not establish an operational biological misuse pathway.", "zh": "汇总科研表现不能证明实际生物滥用路径。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "cbrn", "questionId": "task-uplift", "relation": "enabling", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§3.2; Appendix A.4" } ] }, { "id": "saferai-glm52-swe-bench-pro", "protocolId": "swe-bench-pro", "name": "SWE-Bench Pro public tasks", "version": "v1.1 · revised 26 August 2026", "sourceId": "saferai-glm52-source", "locator": "§4.2; Appendix A.5", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates repository-level changes; public subsets, scaffolds and reward-hacking scans are version-specific.", "zh": "评测仓库级代码修改；公开子集、脚手架和奖励投机扫描有各自版本。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "General coding capability is enabling evidence, not direct evidence of loss of control.", "zh": "一般编程能力属于使能证据，不是失控的直接证据。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "sustained-autonomous-work", "relation": "enabling", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "§4.2; Appendix A.5" } ] }, { "id": "saferai-glm52-agentic-misalignment", "protocolId": "agentic-misalignment", "name": "Agentic Misalignment with neutral emails", "version": "v1.1 · revised 26 August 2026", "sourceId": "saferai-glm52-source", "locator": "§4.3; Appendix A.9", "primaryDimension": "propensity", "measurementTarget": { "en": "Original agentic-misalignment scenarios extended with 11 neutral emails; 30 samples per threat-by-goal condition.", "zh": "原始情境增加 11 封中性邮件；每个威胁与目标组合 30 个样本。" }, "taskScope": { "en": "Original agentic-misalignment scenarios extended with 11 neutral emails; 30 samples per threat-by-goal condition.", "zh": "原始情境增加 11 封中性邮件；每个威胁与目标组合 30 个样本。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "No real people or systems were harmed; the study does not estimate prevalence in deployment.", "zh": "试验未对真实人员或系统造成损害，也不估计部署环境中的发生率。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "unsanctioned-action", "relation": "direct", "rationale": { "en": "Varied controlled scenarios test actions contrary to the assigned benign role.", "zh": "变化的受控情境测试与既定良性职责相违背的行动。" }, "locator": "§4.3; Appendix A.9" }, { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "Some scenarios test concealed harmful actions in pursuit of an assigned goal.", "zh": "部分情境测试在追求指定目标时隐蔽实施有害行动。" }, "locator": "§4.3; Appendix A.9" } ] }, { "id": "saferai-glm52-mask", "protocolId": "mask", "name": "MASK by archetype", "version": "v1.1 · revised 26 August 2026", "sourceId": "saferai-glm52-source", "locator": "§5.1; Appendix A.7; Table A.3", "primaryDimension": "propensity", "measurementTarget": { "en": "Six MASK archetypes; GLM runs by SaferAI, Claude/GPT comparison results reused from CAIS.", "zh": "六类 MASK 情境；GLM 由 SaferAI 运行，Claude/GPT 对照结果引用 CAIS。" }, "taskScope": { "en": "Six MASK archetypes; GLM runs by SaferAI, Claude/GPT comparison results reused from CAIS.", "zh": "六类 MASK 情境；GLM 由 SaferAI 运行，Claude/GPT 对照结果引用 CAIS。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "§5.1; Appendix A.7; Table A.3" } ] }, { "id": "saferai-glm52-ape", "protocolId": "ape", "name": "APE Turn-1 attempts", "version": "v1.1 · revised 26 August 2026", "sourceId": "saferai-glm52-source", "locator": "§5.2; Appendix A.8", "primaryDimension": "propensity", "measurementTarget": { "en": "Frequency of persuasive attempts under specified conversational conditions.", "zh": "特定对话条件下说服尝试的发生率。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Simulated attempts are not measured human belief change or large-scale disinformation reach.", "zh": "模拟尝试不等于已测量的人类信念变化或大规模虚假信息传播。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "disinformation", "questionId": "persuasion-attempts", "relation": "direct", "rationale": { "en": "The study explicitly varies conditions and scores willingness to attempt persuasion.", "zh": "研究明确变化条件并评估尝试说服的意愿。" }, "locator": "§5.2; Appendix A.8" } ] }, { "id": "saferai-glm52-petri", "protocolId": "petri", "name": "Petri 3.0 automated audit", "version": "v1.1 · revised 26 August 2026", "sourceId": "saferai-glm52-source", "locator": "§6; Appendix A.6", "primaryDimension": "behavior", "measurementTarget": { "en": "Petri 3.0, 173 public seeds, Opus 4.7 auditor and judge; filtered-out scored cases differ by model.", "zh": "Petri 3.0、173 个公开种子、Opus 4.7 审计者和判别器；被过滤的评分案例因模型不同。" }, "taskScope": { "en": "Petri 3.0, 173 public seeds, Opus 4.7 auditor and judge; filtered-out scored cases differ by model.", "zh": "Petri 3.0、173 个公开种子、Opus 4.7 审计者和判别器；被过滤的评分案例因模型不同。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Low average audit scores do not rule out rare severe cases; results depend on seeds, auditor, judge and target configuration.", "zh": "较低平均审计分数不排除罕见严重案例；结果依赖种子、审计者、判别器和目标配置。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-2025-cybench", "protocolId": "cybench", "name": "Cybench", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§4.1.1", "primaryDimension": "capability", "measurementTarget": { "en": "Task completion on 40 professional CTF problems.", "zh": "40 项专业 CTF 任务的完成情况。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Competition tasks and supplied guidance differ from operational attacks.", "zh": "竞赛任务与提供的提示不同于现实攻击条件。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "§4.1.1" } ] }, { "id": "caisi-deepseek-2025-cybergym", "protocolId": "cybergym", "name": "CyberGym", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§4.1.2", "primaryDimension": "capability", "measurementTarget": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Vulnerability reproduction is narrower than autonomous compromise of deployed systems.", "zh": "漏洞复现的范围小于对部署系统的自主入侵。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "vulnerability-reproduction", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "§4.1.2" } ] }, { "id": "caisi-deepseek-2025-ctf-archive", "protocolId": null, "name": "CTF-Archive", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§4.1.3", "primaryDimension": "capability", "measurementTarget": { "en": "CTF-Archive in the report-defined setting.", "zh": "报告定义条件中的 CTF-Archive。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-2025-swe-verified", "protocolId": null, "name": "SWE-Bench Verified", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§4.2.1", "primaryDimension": "capability", "measurementTarget": { "en": "SWE-Bench Verified in the report-defined setting.", "zh": "报告定义条件中的 SWE-Bench Verified。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-2025-breakpoint", "protocolId": null, "name": "Breakpoint", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§4.2.2", "primaryDimension": "capability", "measurementTarget": { "en": "Breakpoint in the report-defined setting.", "zh": "报告定义条件中的 Breakpoint。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-2025-mmlu-pro", "protocolId": null, "name": "MMLU-Pro", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§4.3.1", "primaryDimension": "capability", "measurementTarget": { "en": "MMLU-Pro in the report-defined setting.", "zh": "报告定义条件中的 MMLU-Pro。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-2025-mmmlu", "protocolId": null, "name": "MMMLU", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§4.3.2", "primaryDimension": "capability", "measurementTarget": { "en": "MMMLU in the report-defined setting.", "zh": "报告定义条件中的 MMMLU。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-2025-gpqa", "protocolId": null, "name": "GPQA-Diamond", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§4.3.3", "primaryDimension": "capability", "measurementTarget": { "en": "GPQA-Diamond in the report-defined setting.", "zh": "报告定义条件中的 GPQA-Diamond。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-2025-healthbench", "protocolId": null, "name": "HealthBench", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§4.3.4", "primaryDimension": "capability", "measurementTarget": { "en": "HealthBench in the report-defined setting.", "zh": "报告定义条件中的 HealthBench。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-2025-hle", "protocolId": null, "name": "Humanity’s Last Exam", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§4.3.5", "primaryDimension": "capability", "measurementTarget": { "en": "Humanity’s Last Exam in the report-defined setting.", "zh": "报告定义条件中的 Humanity’s Last Exam。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-2025-smt2025", "protocolId": null, "name": "SMT 2025", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§4.4", "primaryDimension": "capability", "measurementTarget": { "en": "SMT 2025 in the report-defined setting.", "zh": "报告定义条件中的 SMT 2025。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-2025-otis-aime", "protocolId": null, "name": "OTIS-AIME 2025", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§4.4", "primaryDimension": "capability", "measurementTarget": { "en": "OTIS-AIME 2025 in the report-defined setting.", "zh": "报告定义条件中的 OTIS-AIME 2025。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-2025-pumac", "protocolId": null, "name": "PUMaC 2024", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§4.4", "primaryDimension": "capability", "measurementTarget": { "en": "PUMaC 2024 in the report-defined setting.", "zh": "报告定义条件中的 PUMaC 2024。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-2025-agentdojo", "protocolId": "agentdojo", "name": "Agent hijacking", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§5.1", "primaryDimension": "behavior", "measurementTarget": { "en": "Targeted attack success and user-task utility in a tool-use environment.", "zh": "工具环境中的定向攻击成功率和用户任务完成情况。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Attack success must be read alongside benign utility and the tested attack setup.", "zh": "攻击成功率需结合正常任务表现与攻击设置解释。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "agent-compromise", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "§5.1" } ] }, { "id": "caisi-deepseek-2025-jailbreak-robustness", "protocolId": null, "name": "Jailbreak robustness", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§5.2", "primaryDimension": "behavior", "measurementTarget": { "en": "Jailbreak robustness in the report-defined setting.", "zh": "报告定义条件中的 Jailbreak robustness。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-2025-censorship", "protocolId": null, "name": "Censorship and narrative evaluations", "version": "30 September 2025", "sourceId": "caisi-deepseek-2025-source", "locator": "§6", "primaryDimension": "behavior", "measurementTarget": { "en": "Censorship and narrative evaluations in the report-defined setting.", "zh": "报告定义条件中的 Censorship and narrative evaluations。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-v4-ctf-archive-diamond", "protocolId": "ctf-archive-diamond", "name": "CTF-Archive-Diamond", "version": "1 May 2026", "sourceId": "caisi-deepseek-v4-source", "locator": "Capability Results; Figure 2", "primaryDimension": "capability", "measurementTarget": { "en": "285 difficult CTF tasks; Inspect ReAct with 1M weighted tokens. DeepSeek result is IRT-imputed from a subset.", "zh": "285 项高难度 CTF 任务；Inspect ReAct、1M 加权 Token。DeepSeek 结果由子集经 IRT 插补。" }, "taskScope": { "en": "285 difficult CTF tasks; Inspect ReAct with 1M weighted tokens. DeepSeek result is IRT-imputed from a subset.", "zh": "285 项高难度 CTF 任务；Inspect ReAct、1M 加权 Token。DeepSeek 结果由子集经 IRT 插补。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "CTF tasks are not live operational attacks; subset-imputed values are not directly observed full-set success rates.", "zh": "CTF 不是现实行动攻击；子集插补值不是全量直接观察的成功率。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "direct", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "Capability Results; Figure 2" } ] }, { "id": "caisi-deepseek-v4-swe-verified", "protocolId": null, "name": "SWE-Bench Verified", "version": "1 May 2026", "sourceId": "caisi-deepseek-v4-source", "locator": "Capability Results; Figure 2", "primaryDimension": "capability", "measurementTarget": { "en": "SWE-Bench Verified in the report-defined setting.", "zh": "报告定义条件中的 SWE-Bench Verified。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-v4-portbench", "protocolId": "portbench", "name": "PortBench", "version": "1 May 2026", "sourceId": "caisi-deepseek-v4-source", "locator": "Capability Results; Figure 2", "primaryDimension": "capability", "measurementTarget": { "en": "CAISI’s held-out software engineering evaluation uses reference implementations and a reported agent budget.", "zh": "CAISI 的保留软件工程评测采用参考实现和明确智能体预算。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "The task set is non-public and continuous scoring differs from binary task success.", "zh": "任务集未公开，连续评分不同于二元任务成功。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "sustained-autonomous-work", "relation": "enabling", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "Capability Results; Figure 2" } ] }, { "id": "caisi-deepseek-v4-frontierscience", "protocolId": null, "name": "FrontierScience", "version": "1 May 2026", "sourceId": "caisi-deepseek-v4-source", "locator": "Capability Results; Figure 2", "primaryDimension": "capability", "measurementTarget": { "en": "FrontierScience in the report-defined setting.", "zh": "报告定义条件中的 FrontierScience。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-v4-gpqa", "protocolId": null, "name": "GPQA-Diamond", "version": "1 May 2026", "sourceId": "caisi-deepseek-v4-source", "locator": "Capability Results; Figure 2", "primaryDimension": "capability", "measurementTarget": { "en": "GPQA-Diamond in the report-defined setting.", "zh": "报告定义条件中的 GPQA-Diamond。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-v4-arc-agi-2", "protocolId": null, "name": "ARC-AGI-2 semi-private", "version": "1 May 2026", "sourceId": "caisi-deepseek-v4-source", "locator": "Capability Results; Figure 2", "primaryDimension": "capability", "measurementTarget": { "en": "ARC-AGI-2 semi-private in the report-defined setting.", "zh": "报告定义条件中的 ARC-AGI-2 semi-private。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-v4-otis-aime", "protocolId": null, "name": "OTIS-AIME-2025", "version": "1 May 2026", "sourceId": "caisi-deepseek-v4-source", "locator": "Capability Results; Figure 2", "primaryDimension": "capability", "measurementTarget": { "en": "OTIS-AIME-2025 in the report-defined setting.", "zh": "报告定义条件中的 OTIS-AIME-2025。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-v4-pumac", "protocolId": null, "name": "PUMaC 2024", "version": "1 May 2026", "sourceId": "caisi-deepseek-v4-source", "locator": "Capability Results; Figure 2", "primaryDimension": "capability", "measurementTarget": { "en": "PUMaC 2024 in the report-defined setting.", "zh": "报告定义条件中的 PUMaC 2024。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "caisi-deepseek-v4-smt2025", "protocolId": null, "name": "SMT 2025", "version": "1 May 2026", "sourceId": "caisi-deepseek-v4-source", "locator": "Capability Results; Figure 2", "primaryDimension": "capability", "measurementTarget": { "en": "SMT 2025 in the report-defined setting.", "zh": "报告定义条件中的 SMT 2025。" }, "taskScope": { "en": "Source-defined task set; do not substitute the original benchmark population.", "zh": "来源定义的任务集，不能替代原始基准总体。" }, "scoring": { "en": "As specified in the report section. Values from charts are not estimated.", "zh": "按报告章节规定评分，不从图形估算数值。" }, "configurationNote": { "en": "See report-specific harness, tool, budget and judge configuration.", "zh": "见报告特定 Harness、工具、预算和判别器配置。" }, "limitations": { "en": "Exploratory or report-specific setting; not a reusable benchmark claim.", "zh": "探索性或报告特定设置，不据此称为可复用基准。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [] }, { "id": "metr-horizon-11-protocol", "protocolId": "time-horizons", "name": "METR Time Horizon 1.1", "version": "799cc9c4b4483a93fc3445623a49ea1bd74fdeb2", "sourceId": "metr-horizon-11-data", "locator": "results.*; p50_horizon_length / p80_horizon_length; release_date; scaffolds", "primaryDimension": "capability", "measurementTarget": { "en": "Estimated task difficulty at 50% and 80% success, measured in human-expert minutes.", "zh": "以人类专家分钟数计量、成功概率为 50% 和 80% 时的估计任务难度。" }, "taskScope": { "en": "Software, ML and cyber tasks with human-expert duration baselines; task weights and scaffolds follow the source.", "zh": "采用人类专家任务时长基线的软件、机器学习和网络任务；任务权重和脚手架遵循来源。" }, "scoring": { "en": "Source-fitted horizon at 50% or 80% predicted success; uncertainty is source-reported.", "zh": "来源拟合的 50% 或 80% 预测成功任务跨度；不确定性采用原文。" }, "configurationNote": { "en": "Scaffolds differ by model and task; the timeline does not assert matched-budget comparability.", "zh": "脚手架因模型和任务不同；时间线不宣称匹配预算可比。" }, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" }, "scenarioScope": "single-agent", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "sustained-autonomous-work", "relation": "enabling", "rationale": { "en": "The source measures a prerequisite capability; it does not show the risk behavior.", "zh": "来源测量前提能力，并未观察到对应风险行为。" }, "locator": "results.*; p50_horizon_length / p80_horizon_length; release_date; scaffolds" } ] }, { "id": "anthropic-claude-opus-4-5-system-card-cybench", "protocolId": "cybench", "name": "Cybench", "version": "2025-12-05", "sourceId": "official-anthropic-claude-opus-4-5-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Task completion on 40 professional CTF problems.", "zh": "40 项专业 CTF 任务的完成情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-opus-4-5-system-card-cybergym", "protocolId": "cybergym", "name": "CyberGym", "version": "2025-12-05", "sourceId": "official-anthropic-claude-opus-4-5-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "vulnerability-reproduction", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-opus-4-5-system-card-shade-arena", "protocolId": "shade-arena", "name": "SHADE-Arena", "version": "2025-12-05", "sourceId": "official-anthropic-claude-opus-4-5-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "The protocol tests whether agents can complete a covert objective while doing legitimate work.", "zh": "协议测试智能体能否在执行正当任务时完成隐蔽目标。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-opus-4-5-system-card-petri", "protocolId": "petri", "name": "Petri", "version": "2025-12-05", "sourceId": "official-anthropic-claude-opus-4-5-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "behavior", "measurementTarget": { "en": "An auditor explores seeded multi-turn interactions, with rubric-based judging of target transcripts.", "zh": "审计者探索以种子设定启动的多轮交互，并按量表判读目标轨迹。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "anthropic-claude-opus-4-6-system-card-retry-cybench", "protocolId": "cybench", "name": "Cybench", "version": "2026-03-06", "sourceId": "official-anthropic-claude-opus-4-6-system-card-retry", "locator": "PDF contents → named RSP, agentic safety and alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Task completion on 40 professional CTF problems.", "zh": "40 项专业 CTF 任务的完成情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF contents → named RSP, agentic safety and alignment sections" } ] }, { "id": "anthropic-claude-opus-4-6-system-card-retry-cybergym", "protocolId": "cybergym", "name": "CyberGym", "version": "2026-03-06", "sourceId": "official-anthropic-claude-opus-4-6-system-card-retry", "locator": "PDF contents → named RSP, agentic safety and alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "vulnerability-reproduction", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF contents → named RSP, agentic safety and alignment sections" } ] }, { "id": "anthropic-claude-opus-4-6-system-card-retry-mask", "protocolId": "mask", "name": "MASK", "version": "2026-03-06", "sourceId": "official-anthropic-claude-opus-4-6-system-card-retry", "locator": "PDF contents → named RSP, agentic safety and alignment sections", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "PDF contents → named RSP, agentic safety and alignment sections" } ] }, { "id": "anthropic-claude-opus-4-6-system-card-retry-shade-arena", "protocolId": "shade-arena", "name": "SHADE-Arena", "version": "2026-03-06", "sourceId": "official-anthropic-claude-opus-4-6-system-card-retry", "locator": "PDF contents → named RSP, agentic safety and alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "The protocol tests whether agents can complete a covert objective while doing legitimate work.", "zh": "协议测试智能体能否在执行正当任务时完成隐蔽目标。" }, "locator": "PDF contents → named RSP, agentic safety and alignment sections" } ] }, { "id": "anthropic-claude-sonnet-4-6-system-card-cybench", "protocolId": "cybench", "name": "Cybench", "version": "2026-03-06", "sourceId": "official-anthropic-claude-sonnet-4-6-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Task completion on 40 professional CTF problems.", "zh": "40 项专业 CTF 任务的完成情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-sonnet-4-6-system-card-cybergym", "protocolId": "cybergym", "name": "CyberGym", "version": "2026-03-06", "sourceId": "official-anthropic-claude-sonnet-4-6-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "vulnerability-reproduction", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-sonnet-4-6-system-card-shade-arena", "protocolId": "shade-arena", "name": "SHADE-Arena", "version": "2026-03-06", "sourceId": "official-anthropic-claude-sonnet-4-6-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "The protocol tests whether agents can complete a covert objective while doing legitimate work.", "zh": "协议测试智能体能否在执行正当任务时完成隐蔽目标。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-sonnet-4-6-system-card-petri", "protocolId": "petri", "name": "Petri", "version": "2026-03-06", "sourceId": "official-anthropic-claude-sonnet-4-6-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "behavior", "measurementTarget": { "en": "An auditor explores seeded multi-turn interactions, with rubric-based judging of target transcripts.", "zh": "审计者探索以种子设定启动的多轮交互，并按量表判读目标轨迹。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "anthropic-claude-opus-4-7-system-card-cybench", "protocolId": "cybench", "name": "Cybench", "version": "2026-04-16", "sourceId": "official-anthropic-claude-opus-4-7-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Task completion on 40 professional CTF problems.", "zh": "40 项专业 CTF 任务的完成情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-opus-4-7-system-card-cybergym", "protocolId": "cybergym", "name": "CyberGym", "version": "2026-04-16", "sourceId": "official-anthropic-claude-opus-4-7-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "vulnerability-reproduction", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-opus-4-7-system-card-mask", "protocolId": "mask", "name": "MASK", "version": "2026-04-16", "sourceId": "official-anthropic-claude-opus-4-7-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-opus-4-7-system-card-shade-arena", "protocolId": "shade-arena", "name": "SHADE-Arena", "version": "2026-04-16", "sourceId": "official-anthropic-claude-opus-4-7-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "The protocol tests whether agents can complete a covert objective while doing legitimate work.", "zh": "协议测试智能体能否在执行正当任务时完成隐蔽目标。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-opus-4-7-system-card-petri", "protocolId": "petri", "name": "Petri", "version": "2026-04-16", "sourceId": "official-anthropic-claude-opus-4-7-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "behavior", "measurementTarget": { "en": "An auditor explores seeded multi-turn interactions, with rubric-based judging of target transcripts.", "zh": "审计者探索以种子设定启动的多轮交互，并按量表判读目标轨迹。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "anthropic-claude-opus-4-8-system-card-cybench", "protocolId": "cybench", "name": "Cybench", "version": "2026-06-17", "sourceId": "official-anthropic-claude-opus-4-8-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Task completion on 40 professional CTF problems.", "zh": "40 项专业 CTF 任务的完成情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-opus-4-8-system-card-cybergym", "protocolId": "cybergym", "name": "CyberGym", "version": "2026-06-17", "sourceId": "official-anthropic-claude-opus-4-8-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "vulnerability-reproduction", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-opus-4-8-system-card-exploitbench", "protocolId": "exploitbench", "name": "ExploitBench", "version": "2026-06-17", "sourceId": "official-anthropic-claude-opus-4-8-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "anthropic-claude-opus-4-8-system-card-mask", "protocolId": "mask", "name": "MASK", "version": "2026-06-17", "sourceId": "official-anthropic-claude-opus-4-8-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-opus-4-8-system-card-shade-arena", "protocolId": "shade-arena", "name": "SHADE-Arena", "version": "2026-06-17", "sourceId": "official-anthropic-claude-opus-4-8-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "The protocol tests whether agents can complete a covert objective while doing legitimate work.", "zh": "协议测试智能体能否在执行正当任务时完成隐蔽目标。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-opus-4-8-system-card-petri", "protocolId": "petri", "name": "Petri", "version": "2026-06-17", "sourceId": "official-anthropic-claude-opus-4-8-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "behavior", "measurementTarget": { "en": "An auditor explores seeded multi-turn interactions, with rubric-based judging of target transcripts.", "zh": "审计者探索以种子设定启动的多轮交互，并按量表判读目标轨迹。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "anthropic-claude-sonnet-5-system-card-cybergym", "protocolId": "cybergym", "name": "CyberGym", "version": "2026-07-10", "sourceId": "official-anthropic-claude-sonnet-5-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "vulnerability-reproduction", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-sonnet-5-system-card-exploitbench", "protocolId": "exploitbench", "name": "ExploitBench", "version": "2026-07-10", "sourceId": "official-anthropic-claude-sonnet-5-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "anthropic-claude-sonnet-5-system-card-mask", "protocolId": "mask", "name": "MASK", "version": "2026-07-10", "sourceId": "official-anthropic-claude-sonnet-5-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-claude-sonnet-5-system-card-shade-arena", "protocolId": "shade-arena", "name": "SHADE-Arena", "version": "2026-07-10", "sourceId": "official-anthropic-claude-sonnet-5-system-card", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "primaryDimension": "capability", "measurementTarget": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "The protocol tests whether agents can complete a covert objective while doing legitimate work.", "zh": "协议测试智能体能否在执行正当任务时完成隐蔽目标。" }, "locator": "PDF table of contents → named RSP / cyber / alignment sections" } ] }, { "id": "anthropic-anthropic-august-2026-risk-report-shade-arena", "protocolId": "shade-arena", "name": "SHADE-Arena", "version": "2026-08-14", "sourceId": "official-anthropic-anthropic-august-2026-risk-report", "locator": "PDF printed pp.17–88", "primaryDimension": "capability", "measurementTarget": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "The protocol tests whether agents can complete a covert objective while doing legitimate work.", "zh": "协议测试智能体能否在执行正当任务时完成隐蔽目标。" }, "locator": "PDF printed pp.17–88" } ] }, { "id": "anthropic-claude-fable-5-1-and-mythos-5-1-system-card-exploitbench", "protocolId": "exploitbench", "name": "ExploitBench", "version": "2026-09-01", "sourceId": "official-anthropic-claude-fable-5-1-and-mythos-5-1-system-card", "locator": "PDF §3", "primaryDimension": "capability", "measurementTarget": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "anthropic-claude-fable-5-1-and-mythos-5-1-system-card-exploitgym", "protocolId": "exploitgym", "name": "ExploitGym", "version": "2026-09-01", "sourceId": "official-anthropic-claude-fable-5-1-and-mythos-5-1-system-card", "locator": "PDF §3", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates exploit construction across real software targets and checks whether success uses the designated vulnerability.", "zh": "在真实软件目标上评测利用构造，并核查成功是否依赖指定漏洞。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "anthropic-claude-fable-5-1-and-mythos-5-1-system-card-mask", "protocolId": "mask", "name": "MASK", "version": "2026-09-01", "sourceId": "official-anthropic-claude-fable-5-1-and-mythos-5-1-system-card", "locator": "PDF §6", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "PDF §6" } ] }, { "id": "anthropic-claude-fable-5-1-and-mythos-5-1-system-card-shade-arena", "protocolId": "shade-arena", "name": "SHADE-Arena", "version": "2026-09-01", "sourceId": "official-anthropic-claude-fable-5-1-and-mythos-5-1-system-card", "locator": "PDF §6", "primaryDimension": "capability", "measurementTarget": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "The protocol tests whether agents can complete a covert objective while doing legitimate work.", "zh": "协议测试智能体能否在执行正当任务时完成隐蔽目标。" }, "locator": "PDF §6" } ] }, { "id": "anthropic-claude-opus-5-5-system-card-exploitbench", "protocolId": "exploitbench", "name": "ExploitBench", "version": "2026-09-22", "sourceId": "official-anthropic-claude-opus-5-5-system-card", "locator": "PDF §3", "primaryDimension": "capability", "measurementTarget": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "anthropic-claude-opus-5-5-system-card-exploitgym", "protocolId": "exploitgym", "name": "ExploitGym", "version": "2026-09-22", "sourceId": "official-anthropic-claude-opus-5-5-system-card", "locator": "PDF §3", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates exploit construction across real software targets and checks whether success uses the designated vulnerability.", "zh": "在真实软件目标上评测利用构造，并核查成功是否依赖指定漏洞。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "anthropic-claude-opus-5-5-system-card-mask", "protocolId": "mask", "name": "MASK", "version": "2026-09-22", "sourceId": "official-anthropic-claude-opus-5-5-system-card", "locator": "PDF §6", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "PDF §6" } ] }, { "id": "anthropic-claude-opus-5-5-system-card-shade-arena", "protocolId": "shade-arena", "name": "SHADE-Arena", "version": "2026-09-22", "sourceId": "official-anthropic-claude-opus-5-5-system-card", "locator": "PDF §6", "primaryDimension": "capability", "measurementTarget": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "The protocol tests whether agents can complete a covert objective while doing legitimate work.", "zh": "协议测试智能体能否在执行正当任务时完成隐蔽目标。" }, "locator": "PDF §6" } ] }, { "id": "anthropic-claude-sonnet-5-5-system-card-exploitbench", "protocolId": "exploitbench", "name": "ExploitBench", "version": "2026-09-28", "sourceId": "official-anthropic-claude-sonnet-5-5-system-card", "locator": "PDF §3", "primaryDimension": "capability", "measurementTarget": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "anthropic-claude-sonnet-5-5-system-card-exploitgym", "protocolId": "exploitgym", "name": "ExploitGym", "version": "2026-09-28", "sourceId": "official-anthropic-claude-sonnet-5-5-system-card", "locator": "PDF §3", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates exploit construction across real software targets and checks whether success uses the designated vulnerability.", "zh": "在真实软件目标上评测利用构造，并核查成功是否依赖指定漏洞。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "anthropic-claude-sonnet-5-5-system-card-mask", "protocolId": "mask", "name": "MASK", "version": "2026-09-28", "sourceId": "official-anthropic-claude-sonnet-5-5-system-card", "locator": "PDF §6", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "PDF §6" } ] }, { "id": "anthropic-claude-sonnet-5-5-system-card-shade-arena", "protocolId": "shade-arena", "name": "SHADE-Arena", "version": "2026-09-28", "sourceId": "official-anthropic-claude-sonnet-5-5-system-card", "locator": "PDF §6", "primaryDimension": "capability", "measurementTarget": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "The protocol tests whether agents can complete a covert objective while doing legitimate work.", "zh": "协议测试智能体能否在执行正当任务时完成隐蔽目标。" }, "locator": "PDF §6" } ] }, { "id": "anthropic-claude-haiku-5-5-system-card-exploitbench", "protocolId": "exploitbench", "name": "ExploitBench", "version": "2026-10-07", "sourceId": "official-anthropic-claude-haiku-5-5-system-card", "locator": "PDF §3", "primaryDimension": "capability", "measurementTarget": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "anthropic-claude-haiku-5-5-system-card-exploitgym", "protocolId": "exploitgym", "name": "ExploitGym", "version": "2026-10-07", "sourceId": "official-anthropic-claude-haiku-5-5-system-card", "locator": "PDF §3", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates exploit construction across real software targets and checks whether success uses the designated vulnerability.", "zh": "在真实软件目标上评测利用构造，并核查成功是否依赖指定漏洞。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "anthropic-claude-haiku-5-5-system-card-mask", "protocolId": "mask", "name": "MASK", "version": "2026-10-07", "sourceId": "official-anthropic-claude-haiku-5-5-system-card", "locator": "PDF §6", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "PDF §6" } ] }, { "id": "anthropic-claude-haiku-5-5-system-card-shade-arena", "protocolId": "shade-arena", "name": "SHADE-Arena", "version": "2026-10-07", "sourceId": "official-anthropic-claude-haiku-5-5-system-card", "locator": "PDF §6", "primaryDimension": "capability", "measurementTarget": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "The protocol tests whether agents can complete a covert objective while doing legitimate work.", "zh": "协议测试智能体能否在执行正当任务时完成隐蔽目标。" }, "locator": "PDF §6" } ] }, { "id": "google-deepmind-gemini-4-argon-safety-security-report-exploitbench", "protocolId": "exploitbench", "name": "ExploitBench", "version": "Official source captured 2026-10-25", "sourceId": "official-google-deepmind-gemini-4-argon-safety-security-report", "locator": "PDF printed pp.21–53", "primaryDimension": "capability", "measurementTarget": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "meta-muse-spark-open-evaluation-methodology-fortress", "protocolId": "fortress", "name": "FORTRESS", "version": "Official source captured 2026-10-25", "sourceId": "official-meta-muse-spark-open-evaluation-methodology", "locator": "Original source → Overall Methodology; safety benchmark descriptions", "primaryDimension": "behavior", "measurementTarget": { "en": "Evaluates safeguards on national-security and public-safety tasks, separating hazardous requests from benign-use utility.", "zh": "在国家安全与公共安全任务上评测防护，区分高危请求与正常用途的有用性。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "meta-muse-spark-open-model-card-fortress", "protocolId": "fortress", "name": "FORTRESS", "version": "Official source captured 2026-10-25", "sourceId": "official-meta-muse-spark-open-model-card", "locator": "Original source → Safety and Preparedness; Considerations and Limitations", "primaryDimension": "behavior", "measurementTarget": { "en": "Evaluates safeguards on national-security and public-safety tasks, separating hazardous requests from benign-use utility.", "zh": "在国家安全与公共安全任务上评测防护，区分高危请求与正常用途的有用性。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "meta-muse-wonder-model-card-cybergym", "protocolId": "cybergym", "name": "CyberGym", "version": "2026-10-23", "sourceId": "official-meta-muse-wonder-model-card", "locator": "PDF pp.7–10", "primaryDimension": "capability", "measurementTarget": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "vulnerability-reproduction", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF pp.7–10" } ] }, { "id": "meta-muse-wonder-model-card-exploitbench", "protocolId": "exploitbench", "name": "ExploitBench", "version": "2026-10-23", "sourceId": "official-meta-muse-wonder-model-card", "locator": "PDF pp.7–10", "primaryDimension": "capability", "measurementTarget": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "meta-muse-wonder-model-card-exploitgym", "protocolId": "exploitgym", "name": "ExploitGym", "version": "2026-10-23", "sourceId": "official-meta-muse-wonder-model-card", "locator": "PDF pp.7–10", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates exploit construction across real software targets and checks whether success uses the designated vulnerability.", "zh": "在真实软件目标上评测利用构造，并核查成功是否依赖指定漏洞。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "meta-muse-wonder-model-card-mask", "protocolId": "mask", "name": "MASK", "version": "2026-10-23", "sourceId": "official-meta-muse-wonder-model-card", "locator": "PDF pp.7–10", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "PDF pp.7–10" } ] }, { "id": "meta-muse-wonder-model-card-agentic-misalignment", "protocolId": "agentic-misalignment", "name": "Agentic Misalignment", "version": "2026-10-23", "sourceId": "official-meta-muse-wonder-model-card", "locator": "PDF pp.7–10", "primaryDimension": "propensity", "measurementTarget": { "en": "Conditional rates and observed actions in controlled simulations.", "zh": "受控模拟中的条件性发生率与可观察行动。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "unsanctioned-action", "relation": "direct", "rationale": { "en": "Varied controlled scenarios test actions contrary to the assigned benign role.", "zh": "变化的受控情境测试与既定良性职责相违背的行动。" }, "locator": "PDF pp.7–10" }, { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "Some scenarios test concealed harmful actions in pursuit of an assigned goal.", "zh": "部分情境测试在追求指定目标时隐蔽实施有害行动。" }, "locator": "PDF pp.7–10" } ] }, { "id": "meta-muse-spark-safety-preparedness-report-cybench", "protocolId": "cybench", "name": "Cybench", "version": "2026-08-24", "sourceId": "official-meta-muse-spark-safety-preparedness-report", "locator": "https://arxiv.org/html/2606.12429#S2.SS2", "primaryDimension": "capability", "measurementTarget": { "en": "Task completion on 40 professional CTF problems.", "zh": "40 项专业 CTF 任务的完成情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "https://arxiv.org/html/2606.12429#S2.SS2" } ] }, { "id": "meta-muse-spark-safety-preparedness-report-cybergym", "protocolId": "cybergym", "name": "CyberGym", "version": "2026-08-24", "sourceId": "official-meta-muse-spark-safety-preparedness-report", "locator": "https://arxiv.org/html/2606.12429#S2.SS2", "primaryDimension": "capability", "measurementTarget": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "vulnerability-reproduction", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "https://arxiv.org/html/2606.12429#S2.SS2" } ] }, { "id": "meta-muse-spark-safety-preparedness-report-mask", "protocolId": "mask", "name": "MASK", "version": "2026-08-24", "sourceId": "official-meta-muse-spark-safety-preparedness-report", "locator": "https://arxiv.org/html/2606.12429#S2.SS3", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "https://arxiv.org/html/2606.12429#S2.SS3" } ] }, { "id": "meta-muse-spark-safety-preparedness-report-agentic-misalignment", "protocolId": "agentic-misalignment", "name": "Agentic Misalignment", "version": "2026-08-24", "sourceId": "official-meta-muse-spark-safety-preparedness-report", "locator": "https://arxiv.org/html/2606.12429#S2.SS3", "primaryDimension": "propensity", "measurementTarget": { "en": "Conditional rates and observed actions in controlled simulations.", "zh": "受控模拟中的条件性发生率与可观察行动。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "unsanctioned-action", "relation": "direct", "rationale": { "en": "Varied controlled scenarios test actions contrary to the assigned benign role.", "zh": "变化的受控情境测试与既定良性职责相违背的行动。" }, "locator": "https://arxiv.org/html/2606.12429#S2.SS3" }, { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "Some scenarios test concealed harmful actions in pursuit of an assigned goal.", "zh": "部分情境测试在追求指定目标时隐蔽实施有害行动。" }, "locator": "https://arxiv.org/html/2606.12429#S2.SS3" } ] }, { "id": "meta-muse-spark-safety-preparedness-report-agentdojo", "protocolId": "agentdojo", "name": "AgentDojo", "version": "2026-08-24", "sourceId": "official-meta-muse-spark-safety-preparedness-report", "locator": "https://arxiv.org/html/2606.12429#S3.SS2", "primaryDimension": "behavior", "measurementTarget": { "en": "Targeted attack success and user-task utility in a tool-use environment.", "zh": "工具环境中的定向攻击成功率和用户任务完成情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "agent-compromise", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "https://arxiv.org/html/2606.12429#S3.SS2" } ] }, { "id": "meta-muse-spark-safety-preparedness-report-deceptionbench", "protocolId": "deceptionbench", "name": "DeceptionBench", "version": "2026-08-24", "sourceId": "official-meta-muse-spark-safety-preparedness-report", "locator": "https://arxiv.org/html/2606.12429#S4.SS1.SSS5", "primaryDimension": "propensity", "measurementTarget": { "en": "Evaluates 150 authored scenarios across five social domains, including neutral, incentive and pressure conditions.", "zh": "评测五个社会领域的 150 个编写情境，区分中性、激励与压力条件。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "https://arxiv.org/html/2606.12429#S4.SS1.SSS5" } ] }, { "id": "meta-muse-spark-safety-preparedness-report-alignment-faking", "protocolId": "alignment-faking", "name": "Alignment Faking", "version": "2026-08-24", "sourceId": "official-meta-muse-spark-safety-preparedness-report", "locator": "https://arxiv.org/html/2606.12429#S4.SS1.SSS11", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-dependent response differences and reported traces.", "zh": "不同试验条件下的响应差异及所报告轨迹。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "Repeated condition contrasts probe a bounded tendency to present compliant responses while preserving a conflicting objective.", "zh": "重复条件对照测试在目标冲突时表现顺从响应的有界倾向。" }, "locator": "https://arxiv.org/html/2606.12429#S4.SS1.SSS11" } ] }, { "id": "meta-muse-spark-1-1-evaluation-report-cybench", "protocolId": "cybench", "name": "Cybench", "version": "2026-07-09", "sourceId": "official-meta-muse-spark-1-1-evaluation-report", "locator": "PDF §§2.1–2.3", "primaryDimension": "capability", "measurementTarget": { "en": "Task completion on 40 professional CTF problems.", "zh": "40 项专业 CTF 任务的完成情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF §§2.1–2.3" } ] }, { "id": "meta-muse-spark-1-1-evaluation-report-cybergym", "protocolId": "cybergym", "name": "CyberGym", "version": "2026-07-09", "sourceId": "official-meta-muse-spark-1-1-evaluation-report", "locator": "PDF §§2.1–2.3", "primaryDimension": "capability", "measurementTarget": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "vulnerability-reproduction", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF §§2.1–2.3" } ] }, { "id": "meta-muse-spark-1-1-evaluation-report-exploitgym", "protocolId": "exploitgym", "name": "ExploitGym", "version": "2026-07-09", "sourceId": "official-meta-muse-spark-1-1-evaluation-report", "locator": "PDF §§2.1–2.3", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates exploit construction across real software targets and checks whether success uses the designated vulnerability.", "zh": "在真实软件目标上评测利用构造，并核查成功是否依赖指定漏洞。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "meta-muse-spark-1-1-evaluation-report-shade-arena", "protocolId": "shade-arena", "name": "SHADE-Arena", "version": "2026-07-09", "sourceId": "official-meta-muse-spark-1-1-evaluation-report", "locator": "PDF §§2.1–2.3", "primaryDimension": "capability", "measurementTarget": { "en": "Covert side-task completion and monitor detection under defined tool access.", "zh": "在确定工具权限下测量隐蔽副任务完成及监测发现情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "The protocol tests whether agents can complete a covert objective while doing legitimate work.", "zh": "协议测试智能体能否在执行正当任务时完成隐蔽目标。" }, "locator": "PDF §§2.1–2.3" } ] }, { "id": "meta-muse-spark-1-1-evaluation-report-mask", "protocolId": "mask", "name": "MASK", "version": "2026-07-09", "sourceId": "official-meta-muse-spark-1-1-evaluation-report", "locator": "PDF §§2.1–2.3", "primaryDimension": "propensity", "measurementTarget": { "en": "Condition-bounded frequency of lying under stated pressures.", "zh": "明确压力条件下说谎的有界发生率。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "Repeated pressured scenarios distinguish lying from mistaken beliefs.", "zh": "重复压力情境将说谎与错误信念区分。" }, "locator": "PDF §§2.1–2.3" } ] }, { "id": "meta-muse-spark-1-1-evaluation-report-fortress", "protocolId": "fortress", "name": "FORTRESS", "version": "2026-07-09", "sourceId": "official-meta-muse-spark-1-1-evaluation-report", "locator": "PDF §3", "primaryDimension": "behavior", "measurementTarget": { "en": "Evaluates safeguards on national-security and public-safety tasks, separating hazardous requests from benign-use utility.", "zh": "在国家安全与公共安全任务上评测防护，区分高危请求与正常用途的有用性。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "meta-muse-spark-1-1-evaluation-report-agentdojo", "protocolId": "agentdojo", "name": "AgentDojo", "version": "2026-07-09", "sourceId": "official-meta-muse-spark-1-1-evaluation-report", "locator": "PDF §3", "primaryDimension": "behavior", "measurementTarget": { "en": "Targeted attack success and user-task utility in a tool-use environment.", "zh": "工具环境中的定向攻击成功率和用户任务完成情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "agent-compromise", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF §3" } ] }, { "id": "meta-muse-spark-1-1-evaluation-report-deceptionbench", "protocolId": "deceptionbench", "name": "DeceptionBench", "version": "2026-07-09", "sourceId": "official-meta-muse-spark-1-1-evaluation-report", "locator": "PDF §4", "primaryDimension": "propensity", "measurementTarget": { "en": "Evaluates 150 authored scenarios across five social domains, including neutral, incentive and pressure conditions.", "zh": "评测五个社会领域的 150 个编写情境，区分中性、激励与压力条件。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "deception", "questionId": "misleading-communication", "relation": "direct", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "PDF §4" } ] }, { "id": "meta-muse-spark-1-1-evaluation-report-agentic-misalignment", "protocolId": "agentic-misalignment", "name": "Agentic Misalignment", "version": "2026-07-09", "sourceId": "official-meta-muse-spark-1-1-evaluation-report", "locator": "PDF §4", "primaryDimension": "propensity", "measurementTarget": { "en": "Conditional rates and observed actions in controlled simulations.", "zh": "受控模拟中的条件性发生率与可观察行动。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "unsanctioned-action", "relation": "direct", "rationale": { "en": "Varied controlled scenarios test actions contrary to the assigned benign role.", "zh": "变化的受控情境测试与既定良性职责相违背的行动。" }, "locator": "PDF §4" }, { "area": "deception", "questionId": "covert-goal-pursuit", "relation": "direct", "rationale": { "en": "Some scenarios test concealed harmful actions in pursuit of an assigned goal.", "zh": "部分情境测试在追求指定目标时隐蔽实施有害行动。" }, "locator": "PDF §4" } ] }, { "id": "mistral-shieldstral-harmbench", "protocolId": "harmbench", "name": "HarmBench", "version": "Official source captured 2026-10-25", "sourceId": "official-mistral-shieldstral", "locator": "Original source → Safety classification, refusal detection and multimodal safety evaluations", "primaryDimension": "behavior", "measurementTarget": { "en": "Attack-dependent generation of harmful responses.", "zh": "依赖具体攻击方法的有害回答生成情况。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "openai-update-to-gpt-5-system-card-gpt-5-2-mle-bench", "protocolId": "mle-bench", "name": "MLE-bench", "version": "2025-12-11", "sourceId": "official-openai-update-to-gpt-5-system-card-gpt-5-2", "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates autonomous machine-learning engineering using competition tasks and human performance baselines.", "zh": "通过竞赛任务和人类表现基线评测自主机器学习工程能力。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "openai-gpt-5-4-thinking-system-card-mle-bench", "protocolId": "mle-bench", "name": "MLE-bench", "version": "2026-03-05", "sourceId": "official-openai-gpt-5-4-thinking-system-card", "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates autonomous machine-learning engineering using competition tasks and human performance baselines.", "zh": "通过竞赛任务和人类表现基线评测自主机器学习工程能力。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "openai-gpt-5-5-system-card-mle-bench", "protocolId": "mle-bench", "name": "MLE-bench", "version": "2026-04-23", "sourceId": "official-openai-gpt-5-5-system-card", "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates autonomous machine-learning engineering using competition tasks and human performance baselines.", "zh": "通过竞赛任务和人类表现基线评测自主机器学习工程能力。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "openai-gpt-5-6-system-card-cybergym", "protocolId": "cybergym", "name": "CyberGym", "version": "2026-07-09", "sourceId": "official-openai-gpt-5-6-system-card", "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections", "primaryDimension": "capability", "measurementTarget": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "vulnerability-reproduction", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections" } ] }, { "id": "openai-gpt-5-6-system-card-exploitbench", "protocolId": "exploitbench", "name": "ExploitBench", "version": "2026-07-09", "sourceId": "official-openai-gpt-5-6-system-card", "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections", "primaryDimension": "capability", "measurementTarget": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "openai-gpt-5-6-system-card-mle-bench", "protocolId": "mle-bench", "name": "MLE-bench", "version": "2026-07-09", "sourceId": "official-openai-gpt-5-6-system-card", "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates autonomous machine-learning engineering using competition tasks and human performance baselines.", "zh": "通过竞赛任务和人类表现基线评测自主机器学习工程能力。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "openai-gpt-6-astra-system-card-exploitgym", "protocolId": "exploitgym", "name": "ExploitGym", "version": "2026-09-09", "sourceId": "official-openai-gpt-6-astra-system-card", "locator": "https://deploymentsafety.openai.com/gpt-6-astra#section.8", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates exploit construction across real software targets and checks whether success uses the designated vulnerability.", "zh": "在真实软件目标上评测利用构造，并核查成功是否依赖指定漏洞。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "openai-gpt-6-astra-system-card-exploitbench", "protocolId": "exploitbench", "name": "ExploitBench", "version": "2026-09-09", "sourceId": "official-openai-gpt-6-astra-system-card", "locator": "https://deploymentsafety.openai.com/gpt-6-astra#section.10", "primaryDimension": "capability", "measurementTarget": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "openai-gpt-6-astra-system-card-sre-bench", "protocolId": "sre-bench", "name": "SRE-Bench", "version": "2026-09-09", "sourceId": "official-openai-gpt-6-astra-system-card", "locator": "https://deploymentsafety.openai.com/gpt-6-astra#section.10", "primaryDimension": "capability", "measurementTarget": { "en": "Studies agentic reverse engineering with realistic binary-analysis tasks designed to reduce benchmark contamination.", "zh": "通过贴近实际的二进制分析任务研究智能体逆向工程能力，并控制评测污染。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "openai-addendum-to-gpt-6-astra-system-card-gpt-6-1-sol-exploitgym", "protocolId": "exploitgym", "name": "ExploitGym", "version": "2026-09-29", "sourceId": "official-openai-addendum-to-gpt-6-astra-system-card-gpt-6-1-sol", "locator": "https://deploymentsafety.openai.com/gpt-6-1-sol#section.7", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates exploit construction across real software targets and checks whether success uses the designated vulnerability.", "zh": "在真实软件目标上评测利用构造，并核查成功是否依赖指定漏洞。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "openai-addendum-to-gpt-6-astra-system-card-gpt-6-1-sol-exploitbench", "protocolId": "exploitbench", "name": "ExploitBench", "version": "2026-09-29", "sourceId": "official-openai-addendum-to-gpt-6-astra-system-card-gpt-6-1-sol", "locator": "https://deploymentsafety.openai.com/gpt-6-1-sol#section.9", "primaryDimension": "capability", "measurementTarget": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "openai-gpt-6-sol-and-gpt-6-luna-october-2026-update-exploitbench", "protocolId": "exploitbench", "name": "ExploitBench", "version": "2026-10-07", "sourceId": "official-openai-gpt-6-sol-and-gpt-6-luna-october-2026-update", "locator": "https://deploymentsafety.openai.com/gpt-6-october#section.8", "primaryDimension": "capability", "measurementTarget": { "en": "Separates exploit development into capability levels, distinguishing bug triggering from reusable exploitation primitives and control.", "zh": "将漏洞利用拆分为能力层级，区分触发缺陷、构造利用原语和获取控制。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "openai-gpt-6-sol-and-gpt-6-luna-october-2026-update-secbench-pro", "protocolId": "secbench-pro", "name": "SEC-Bench Pro", "version": "2026-10-07", "sourceId": "official-openai-gpt-6-sol-and-gpt-6-luna-october-2026-update", "locator": "https://deploymentsafety.openai.com/gpt-6-october#section.8", "primaryDimension": "capability", "measurementTarget": { "en": "Working proof-of-concept reproduction on a versioned software-security task set.", "zh": "在有版本的软件安全任务集上复现可验证的漏洞触发输入。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "vulnerability-reproduction", "relation": "direct", "rationale": { "en": "The paper evaluates working reproductions of disclosed bugs.", "zh": "论文评测已披露漏洞的可运行复现。" }, "locator": "https://deploymentsafety.openai.com/gpt-6-october#section.8" }, { "area": "cybersecurity", "questionId": "cyber-task-execution", "relation": "enabling", "rationale": { "en": "Vulnerability reproduction informs cyber task capability but not a full intrusion chain.", "zh": "漏洞复现可为网络任务能力提供前提证据，但不等于完整入侵链。" }, "locator": "https://deploymentsafety.openai.com/gpt-6-october#section.8" } ] }, { "id": "qwen-qwen3-8-2-4t-a95b-model-card-paperbench", "protocolId": "paperbench", "name": "PaperBench", "version": "Official source captured 2026-10-25", "sourceId": "official-qwen-qwen3-8-2-4t-a95b-model-card", "locator": "Original source → Benchmark Results; evaluation footnotes", "primaryDimension": "capability", "measurementTarget": { "en": "Rubric-graded research replication tasks with an agent scaffold.", "zh": "在智能体脚手架下完成基于评分规则的研究复现任务。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "sustained-autonomous-work", "relation": "enabling", "rationale": { "en": "Multi-step research replication informs sustained work capacity.", "zh": "多步骤研究复现可反映持续工作能力。" }, "locator": "Original source → Benchmark Results; evaluation footnotes" } ] }, { "id": "xai-grok-4-7-model-card-cybergym", "protocolId": "cybergym", "name": "CyberGym", "version": "2026-09-21", "sourceId": "official-xai-grok-4-7-model-card", "locator": "PDF pp.17–19 (printed 16–18)", "primaryDimension": "capability", "measurementTarget": { "en": "Validated reproduction across a corpus of historical vulnerabilities.", "zh": "历史漏洞集合中的经验证复现表现。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cybersecurity", "questionId": "vulnerability-reproduction", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF pp.17–19 (printed 16–18)" } ] }, { "id": "xai-grok-4-7-model-card-cve-bench", "protocolId": "cve-bench", "name": "CVE-Bench", "version": "2026-09-21", "sourceId": "official-xai-grok-4-7-model-card", "locator": "PDF pp.17–19 (printed 16–18)", "primaryDimension": "capability", "measurementTarget": { "en": "Measures agents’ exploitation of real web-application vulnerabilities in runnable environments with explicit success checks.", "zh": "在可运行环境中，以明确成功判据评测智能体利用真实 Web 应用漏洞的能力。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "xai-grok-4-7-model-card-wmdp", "protocolId": "wmdp", "name": "WMDP", "version": "2026-09-21", "sourceId": "official-xai-grok-4-7-model-card", "locator": "PDF pp.20–23 (printed 19–22)", "primaryDimension": "capability", "measurementTarget": { "en": "Multiple-choice knowledge measurements.", "zh": "选择题形式的知识测量。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cbrn", "questionId": "hazardous-knowledge", "relation": "direct", "rationale": { "en": "The source directly measures the stated construct under its protocol.", "zh": "来源在其协议下直接测量所述构念。" }, "locator": "PDF pp.20–23 (printed 19–22)" } ] }, { "id": "xai-grok-4-7-model-card-lab-bench", "protocolId": "lab-bench", "name": "LAB-Bench", "version": "2026-09-21", "sourceId": "official-xai-grok-4-7-model-card", "locator": "PDF pp.20–23 (printed 19–22)", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates a range of biology research skills with structured questions; text-only or missing-input subsets remain separate protocols.", "zh": "用结构化问题评测生物研究技能；纯文本或缺失输入子集采用独立协议。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "cbrn", "questionId": "task-uplift", "relation": "enabling", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "PDF pp.20–23 (printed 19–22)" } ] }, { "id": "xai-grok-4-7-model-card-strongreject", "protocolId": "strongreject", "name": "StrongREJECT", "version": "2026-09-21", "sourceId": "official-xai-grok-4-7-model-card", "locator": "PDF p.23 (printed 22)", "primaryDimension": "behavior", "measurementTarget": { "en": "A graded response evaluator for jailbreak effectiveness.", "zh": "用于越狱有效性分析的回答评分方法。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [] }, { "id": "zai-glm-5-2-model-card-swe-bench-pro", "protocolId": "swe-bench-pro", "name": "SWE-bench Pro", "version": "Official source captured 2026-10-25", "sourceId": "official-zai-glm-5-2-model-card", "locator": "Original source → Benchmark; Footnote", "primaryDimension": "capability", "measurementTarget": { "en": "Evaluates repository-level changes; public subsets, scaffolds and reward-hacking scans are version-specific.", "zh": "评测仓库级代码修改；公开子集、脚手架和奖励投机扫描有各自版本。" }, "taskScope": { "en": "Developer-report implementation; exact task selection and scoring require separate result-level verification.", "zh": "开发者报告中的实施版本；具体任务选择和评分仍需逐项核验。" }, "scoring": { "en": "See the report section; no score transcribed in this record.", "zh": "见报告相应章节；此档案未抄录分数。" }, "configurationNote": { "en": "Report configurations are not assumed identical to the original Benchmark.", "zh": "不假定报告配置与原始 Benchmark 相同。" }, "limitations": { "en": "A protocol mention does not establish cross-report comparability or independent replication.", "zh": "协议引用不代表跨报告可比，也不代表独立复现。" }, "scenarioScope": "system", "agentConfiguration": null, "riskAssociations": [ { "area": "autonomy", "questionId": "sustained-autonomous-work", "relation": "enabling", "rationale": { "en": "The protocol explicitly measures this question or an enabling sub-capability; conclusions retain its stated conditions.", "zh": "该协议明确测量该问题或其使能子能力，结论保留给定条件。" }, "locator": "Original source → Benchmark; Footnote" } ] } ], "assessmentEvidence": [ { "id": "link-neo-kimi-k3-charxiv-kimi-k3", "modelId": "kimi-k3", "assessmentId": "neo-kimi-k3", "variantId": "neo-kimi-k3-charxiv", "resultId": "neo-kimi-k3-charxiv-kimi-k3", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-09-11", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-kimi-k3-charxiv-gpt-5-6-terra", "modelId": "gpt-5-6-terra", "assessmentId": "neo-kimi-k3", "variantId": "neo-kimi-k3-charxiv", "resultId": "neo-kimi-k3-charxiv-gpt-5-6-terra", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-09-11", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-kimi-k3-charxiv-claude-sonnet-5", "modelId": "claude-sonnet-5", "assessmentId": "neo-kimi-k3", "variantId": "neo-kimi-k3-charxiv", "resultId": "neo-kimi-k3-charxiv-claude-sonnet-5", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-09-11", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-kimi-k3-charxiv-deepseek-v4-flash-0731", "modelId": "deepseek-v4-flash-0731", "assessmentId": "neo-kimi-k3", "variantId": "neo-kimi-k3-charxiv", "resultId": "neo-kimi-k3-charxiv-deepseek-v4-flash-0731", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-09-11", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-kimi-k3-lab-bench-kimi-k3", "modelId": "kimi-k3", "assessmentId": "neo-kimi-k3", "variantId": "neo-kimi-k3-lab-bench", "resultId": "neo-kimi-k3-lab-bench-kimi-k3", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-09-11", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-kimi-k3-lab-bench-gpt-5-6-terra", "modelId": "gpt-5-6-terra", "assessmentId": "neo-kimi-k3", "variantId": "neo-kimi-k3-lab-bench", "resultId": "neo-kimi-k3-lab-bench-gpt-5-6-terra", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-09-11", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-kimi-k3-lab-bench-claude-sonnet-5", "modelId": "claude-sonnet-5", "assessmentId": "neo-kimi-k3", "variantId": "neo-kimi-k3-lab-bench", "resultId": "neo-kimi-k3-lab-bench-claude-sonnet-5", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-09-11", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-kimi-k3-lab-bench-deepseek-v4-flash-0731", "modelId": "deepseek-v4-flash-0731", "assessmentId": "neo-kimi-k3", "variantId": "neo-kimi-k3-lab-bench", "resultId": "neo-kimi-k3-lab-bench-deepseek-v4-flash-0731", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-09-11", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-kimi-k3-agentic-misalignment-kimi-k3", "modelId": "kimi-k3", "assessmentId": "neo-kimi-k3", "variantId": "neo-kimi-k3-agentic-misalignment", "resultId": "neo-kimi-k3-agentic-misalignment-kimi-k3", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-09-11", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-kimi-k3-evalawarebench-kimi-k3", "modelId": "kimi-k3", "assessmentId": "neo-kimi-k3", "variantId": "neo-kimi-k3-evalawarebench", "resultId": "neo-kimi-k3-evalawarebench-kimi-k3", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-09-11", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-kimi-k3-palisade-shutdown-kimi-k3", "modelId": "kimi-k3", "assessmentId": "neo-kimi-k3", "variantId": "neo-kimi-k3-palisade-shutdown", "resultId": "neo-kimi-k3-palisade-shutdown-kimi-k3", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-09-11", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-glm52-mask-glm-5-2", "modelId": "glm-5-2", "assessmentId": "neo-glm52", "variantId": "neo-glm52-mask", "resultId": "neo-glm52-mask-glm-5-2", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-07-03", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-glm52-mask-deepseek-v4-pro", "modelId": "deepseek-v4-pro", "assessmentId": "neo-glm52", "variantId": "neo-glm52-mask", "resultId": "neo-glm52-mask-deepseek-v4-pro", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-07-03", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-glm52-mask-claude-sonnet-4-6", "modelId": "claude-sonnet-4-6", "assessmentId": "neo-glm52", "variantId": "neo-glm52-mask", "resultId": "neo-glm52-mask-claude-sonnet-4-6", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-07-03", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-glm52-deceptionbench-glm-5-2", "modelId": "glm-5-2", "assessmentId": "neo-glm52", "variantId": "neo-glm52-deceptionbench", "resultId": "neo-glm52-deceptionbench-glm-5-2", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-07-03", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-glm52-deceptionbench-deepseek-v4-pro", "modelId": "deepseek-v4-pro", "assessmentId": "neo-glm52", "variantId": "neo-glm52-deceptionbench", "resultId": "neo-glm52-deceptionbench-deepseek-v4-pro", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-07-03", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-glm52-persusafety-glm-5-2", "modelId": "glm-5-2", "assessmentId": "neo-glm52", "variantId": "neo-glm52-persusafety", "resultId": "neo-glm52-persusafety-glm-5-2", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-07-03", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-glm52-persusafety-claude-sonnet-5", "modelId": "claude-sonnet-5", "assessmentId": "neo-glm52", "variantId": "neo-glm52-persusafety", "resultId": "neo-glm52-persusafety-claude-sonnet-5", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-07-03", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-neo-glm52-persusafety-deepseek-v4-pro", "modelId": "deepseek-v4-pro", "assessmentId": "neo-glm52", "variantId": "neo-glm52-persusafety", "resultId": "neo-glm52-persusafety-deepseek-v4-pro", "origin": "external", "executorIds": [ "neo" ], "originalExecutorIds": [], "publicSince": "2026-07-03", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-saferai-glm52-cybergym-glm-5-2", "modelId": "glm-5-2", "assessmentId": "saferai-glm52", "variantId": "saferai-glm52-cybergym", "resultId": "saferai-glm52-cybergym-glm-5-2", "origin": "external", "executorIds": [ "saferai" ], "originalExecutorIds": [], "publicSince": "2026-08-26", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-saferai-glm52-mask-glm-5-2", "modelId": "glm-5-2", "assessmentId": "saferai-glm52", "variantId": "saferai-glm52-mask", "resultId": "saferai-glm52-mask-glm-5-2", "origin": "external", "executorIds": [ "saferai" ], "originalExecutorIds": [], "publicSince": "2026-08-26", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-saferai-glm52-mask-claude-opus-4-7", "modelId": "claude-opus-4-7", "assessmentId": "saferai-glm52", "variantId": "saferai-glm52-mask", "resultId": "saferai-glm52-mask-claude-opus-4-7", "origin": "reused", "executorIds": [ "cais" ], "originalExecutorIds": [ "cais" ], "publicSince": "2026-08-26", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-saferai-glm52-mask-gpt-5-5", "modelId": "gpt-5-5", "assessmentId": "saferai-glm52", "variantId": "saferai-glm52-mask", "resultId": "saferai-glm52-mask-gpt-5-5", "origin": "reused", "executorIds": [ "cais" ], "originalExecutorIds": [ "cais" ], "publicSince": "2026-08-26", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-caisi-deepseek-v4-ctf-archive-diamond-gpt-5-5", "modelId": "gpt-5-5", "assessmentId": "caisi-deepseek-v4", "variantId": "caisi-deepseek-v4-ctf-archive-diamond", "resultId": "caisi-deepseek-v4-ctf-archive-diamond-gpt-5-5", "origin": "external", "executorIds": [ "nist" ], "originalExecutorIds": [], "publicSince": "2026-05-01", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-caisi-deepseek-v4-ctf-archive-diamond-gpt-5-4-mini", "modelId": "gpt-5-4-mini", "assessmentId": "caisi-deepseek-v4", "variantId": "caisi-deepseek-v4-ctf-archive-diamond", "resultId": "caisi-deepseek-v4-ctf-archive-diamond-gpt-5-4-mini", "origin": "external", "executorIds": [ "nist" ], "originalExecutorIds": [], "publicSince": "2026-05-01", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-caisi-deepseek-v4-ctf-archive-diamond-claude-opus-4-6", "modelId": "claude-opus-4-6", "assessmentId": "caisi-deepseek-v4", "variantId": "caisi-deepseek-v4-ctf-archive-diamond", "resultId": "caisi-deepseek-v4-ctf-archive-diamond-claude-opus-4-6", "origin": "external", "executorIds": [ "nist" ], "originalExecutorIds": [], "publicSince": "2026-05-01", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-caisi-deepseek-v4-ctf-archive-diamond-deepseek-v4-pro", "modelId": "deepseek-v4-pro", "assessmentId": "caisi-deepseek-v4", "variantId": "caisi-deepseek-v4-ctf-archive-diamond", "resultId": "caisi-deepseek-v4-ctf-archive-diamond-deepseek-v4-pro", "origin": "external", "executorIds": [ "nist" ], "originalExecutorIds": [], "publicSince": "2026-05-01", "availability": "report-date", "measurement": "imputed", "measurementNote": { "en": "CAISI imputes this value from a subset via IRT; it is not a directly measured full-set success rate.", "zh": "CAISI 根据子集通过 IRT 插补；不是全量直接测量成功率。" } }, { "id": "link-caisi-deepseek-v4-portbench-gpt-5-5", "modelId": "gpt-5-5", "assessmentId": "caisi-deepseek-v4", "variantId": "caisi-deepseek-v4-portbench", "resultId": "caisi-deepseek-v4-portbench-gpt-5-5", "origin": "external", "executorIds": [ "nist" ], "originalExecutorIds": [], "publicSince": "2026-05-01", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-caisi-deepseek-v4-portbench-gpt-5-4-mini", "modelId": "gpt-5-4-mini", "assessmentId": "caisi-deepseek-v4", "variantId": "caisi-deepseek-v4-portbench", "resultId": "caisi-deepseek-v4-portbench-gpt-5-4-mini", "origin": "external", "executorIds": [ "nist" ], "originalExecutorIds": [], "publicSince": "2026-05-01", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-caisi-deepseek-v4-portbench-claude-opus-4-6", "modelId": "claude-opus-4-6", "assessmentId": "caisi-deepseek-v4", "variantId": "caisi-deepseek-v4-portbench", "resultId": "caisi-deepseek-v4-portbench-claude-opus-4-6", "origin": "external", "executorIds": [ "nist" ], "originalExecutorIds": [], "publicSince": "2026-05-01", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-caisi-deepseek-v4-portbench-deepseek-v4-pro", "modelId": "deepseek-v4-pro", "assessmentId": "caisi-deepseek-v4", "variantId": "caisi-deepseek-v4-portbench", "resultId": "caisi-deepseek-v4-portbench-deepseek-v4-pro", "origin": "external", "executorIds": [ "nist" ], "originalExecutorIds": [], "publicSince": "2026-05-01", "availability": "report-date", "measurement": "observed", "measurementNote": null }, { "id": "link-metr-horizon11-claude-3-5-sonnet-20240620-inspect", "modelId": "claude-3-5-sonnet-june-2024", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-claude-3-5-sonnet-20240620-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-claude-3-5-sonnet-20241022-inspect", "modelId": "claude-3-5-sonnet-october-2024", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-claude-3-5-sonnet-20241022-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-claude-3-7-sonnet-inspect", "modelId": "claude-3-7-sonnet", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-claude-3-7-sonnet-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-claude-3-opus-inspect", "modelId": "claude-3-opus", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-claude-3-opus-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-claude-4-1-opus-inspect", "modelId": "claude-opus-4-1", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-claude-4-1-opus-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-claude-4-opus-inspect", "modelId": "claude-opus-4", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-claude-4-opus-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-claude-opus-4-5-inspect", "modelId": "claude-opus-4-5", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-claude-opus-4-5-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-claude-opus-4-6-inspect", "modelId": "claude-opus-4-6", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-claude-opus-4-6-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-davinci-002", "modelId": "gpt-3-davinci-002", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-davinci-002", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-gpt-5-3-codex", "modelId": "gpt-5-3-codex", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-gpt-5-3-codex", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-gemini-3-pro", "modelId": "gemini-3-pro", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-gemini-3-pro", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-gemini-3-1-pro", "modelId": "gemini-3-1-pro", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-gemini-3-1-pro", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-gpt2", "modelId": "gpt-2", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-gpt2", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-gpt-3-5-turbo-instruct", "modelId": "gpt-3-5-turbo-instruct", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-gpt-3-5-turbo-instruct", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-gpt-4", "modelId": "gpt-4", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-gpt-4", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-gpt-4-1106-inspect", "modelId": "gpt-4-november-2023", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-gpt-4-1106-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-gpt-4-turbo-inspect", "modelId": "gpt-4-turbo", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-gpt-4-turbo-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-gpt-4o-inspect", "modelId": "gpt-4o", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-gpt-4o-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-gpt-5-1-codex-max-inspect", "modelId": "gpt-5-1-codex-max", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-gpt-5-1-codex-max-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-gpt-5-2", "modelId": "gpt-5-2", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-gpt-5-2", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-gpt-5-2025-08-07-inspect", "modelId": "gpt-5", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-gpt-5-2025-08-07-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-gpt-5-4", "modelId": "gpt-5-4", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-gpt-5-4", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-o1-inspect", "modelId": "o1", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-o1-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-o1-preview", "modelId": "o1-preview", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-o1-preview", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-o3-inspect", "modelId": "o3", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-o3-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } }, { "id": "link-metr-horizon11-claude-mythos-preview-early-inspect", "modelId": "claude-mythos-preview-early", "assessmentId": "metr-horizon-11", "variantId": "metr-horizon-11-protocol", "resultId": "metr-horizon11-claude-mythos-preview-early-inspect", "origin": "external", "executorIds": [ "metr" ], "originalExecutorIds": [], "publicSince": "2026-10-25", "availability": "first-verified", "measurement": "source-estimate", "measurementNote": { "en": "Source-fitted horizon, with reported bootstrap intervals. First public availability of this numerical version is not established.", "zh": "来源拟合的任务跨度和报告的 Bootstrap 区间；未确定该数值版本首次公开日期。" } } ], "trendSeries": [ { "id": "metr-horizon-1-1", "title": { "en": "Task-completion horizons", "zh": "任务完成跨度" }, "area": "autonomy", "dimension": "capability", "metricIds": [ "metr-p50", "metr-p80" ], "variantIds": [ "metr-horizon-11-protocol" ], "evidenceLinkIds": [ "link-metr-horizon11-claude-3-5-sonnet-20240620-inspect", "link-metr-horizon11-claude-3-5-sonnet-20241022-inspect", "link-metr-horizon11-claude-3-7-sonnet-inspect", "link-metr-horizon11-claude-3-opus-inspect", "link-metr-horizon11-claude-4-1-opus-inspect", "link-metr-horizon11-claude-4-opus-inspect", "link-metr-horizon11-claude-opus-4-5-inspect", "link-metr-horizon11-claude-opus-4-6-inspect", "link-metr-horizon11-davinci-002", "link-metr-horizon11-gpt-5-3-codex", "link-metr-horizon11-gemini-3-pro", "link-metr-horizon11-gemini-3-1-pro", "link-metr-horizon11-gpt2", "link-metr-horizon11-gpt-3-5-turbo-instruct", "link-metr-horizon11-gpt-4", "link-metr-horizon11-gpt-4-1106-inspect", "link-metr-horizon11-gpt-4-turbo-inspect", "link-metr-horizon11-gpt-4o-inspect", "link-metr-horizon11-gpt-5-1-codex-max-inspect", "link-metr-horizon11-gpt-5-2", "link-metr-horizon11-gpt-5-2025-08-07-inspect", "link-metr-horizon11-gpt-5-4", "link-metr-horizon11-o1-inspect", "link-metr-horizon11-o1-preview", "link-metr-horizon11-o3-inspect", "link-metr-horizon11-claude-mythos-preview-early-inspect" ], "connection": "points", "description": { "en": "Task difficulty in human-expert minutes, plotted against public model release. These capability measurements are relevant to autonomy research; they are not a risk probability.", "zh": "以人类专家分钟数表示任务难度，按模型公开时间排列。该能力测量与自主性研究相关，不是风险概率。" }, "limitations": { "en": "Human task difficulty, not elapsed autonomous operation. Values above 16 hours are unreliable with the current suite.", "zh": "衡量人类任务难度，而非自主运行时长。当前任务集下高于 16 小时的测量不可靠。" }, "sourceIds": [ "metr-horizon-11-data", "metr-horizon-11-page" ] }, { "id": "neo-kimi-k3-charxiv", "title": { "en": "CharXiv with images withheld", "zh": "CharXiv with images withheld" }, "area": "deception", "dimension": "behavior", "metricIds": [ "neo-kimi-k3-charxiv-m0" ], "variantIds": [ "neo-kimi-k3-charxiv" ], "evidenceLinkIds": [ "link-neo-kimi-k3-charxiv-kimi-k3", "link-neo-kimi-k3-charxiv-gpt-5-6-terra", "link-neo-kimi-k3-charxiv-claude-sonnet-5", "link-neo-kimi-k3-charxiv-deepseek-v4-flash-0731" ], "connection": "points", "description": { "en": "Source-specific results by model release. Protocol and evaluator scope remain explicit.", "zh": "按模型发布时间展示来源特定结果，保留协议与评测方范围。" }, "limitations": { "en": "The original visual capability benchmark and the missing-image behavioral test measure different constructs.", "zh": "原始视觉能力基准与缺失图像行为测试测量不同构念。" }, "sourceIds": [ "neo-kimi-k3-source" ] }, { "id": "neo-glm52-deceptionbench", "title": { "en": "DeceptionBench", "zh": "DeceptionBench" }, "area": "deception", "dimension": "propensity", "metricIds": [ "neo-glm52-deceptionbench-m0", "neo-glm52-deceptionbench-m1" ], "variantIds": [ "neo-glm52-deceptionbench" ], "evidenceLinkIds": [ "link-neo-glm52-deceptionbench-glm-5-2", "link-neo-glm52-deceptionbench-deepseek-v4-pro" ], "connection": "points", "description": { "en": "Source-specific results by model release. Protocol and evaluator scope remain explicit.", "zh": "按模型发布时间展示来源特定结果，保留协议与评测方范围。" }, "limitations": { "en": "Authored scenarios and model-based judging limit generalization; elicited deception does not establish covert long-term scheming.", "zh": "编写情境和模型判别限制外推；诱发的欺骗不证明隐蔽长程谋划。" }, "sourceIds": [ "neo-glm52-source" ] }, { "id": "neo-glm52-persusafety", "title": { "en": "PersuSafety adapted free-generation judge", "zh": "PersuSafety adapted free-generation judge" }, "area": "disinformation", "dimension": "propensity", "metricIds": [ "neo-glm52-persusafety-m0", "neo-glm52-persusafety-m1" ], "variantIds": [ "neo-glm52-persusafety" ], "evidenceLinkIds": [ "link-neo-glm52-persusafety-glm-5-2", "link-neo-glm52-persusafety-claude-sonnet-5", "link-neo-glm52-persusafety-deepseek-v4-pro" ], "connection": "points", "description": { "en": "Source-specific results by model release. Protocol and evaluator scope remain explicit.", "zh": "按模型发布时间展示来源特定结果，保留协议与评测方范围。" }, "limitations": { "en": "Willingness or simulated dialogue scores do not directly measure changes in human beliefs.", "zh": "参与意愿或模拟对话评分不直接测量人的信念变化。" }, "sourceIds": [ "neo-glm52-source" ] }, { "id": "saferai-glm52-cybergym", "title": { "en": "CyberGym hard subset with token budgets", "zh": "CyberGym hard subset with token budgets" }, "area": "cybersecurity", "dimension": "capability", "metricIds": [ "saferai-glm52-cybergym-m0", "saferai-glm52-cybergym-m1" ], "variantIds": [ "saferai-glm52-cybergym" ], "evidenceLinkIds": [ "link-saferai-glm52-cybergym-glm-5-2" ], "connection": "points", "description": { "en": "Source-specific results by model release. Protocol and evaluator scope remain explicit.", "zh": "按模型发布时间展示来源特定结果，保留协议与评测方范围。" }, "limitations": { "en": "Vulnerability reproduction is narrower than autonomous compromise of deployed systems.", "zh": "漏洞复现的范围小于对部署系统的自主入侵。" }, "sourceIds": [ "saferai-glm52-source" ] }, { "id": "saferai-glm52-mask", "title": { "en": "MASK by archetype", "zh": "MASK by archetype" }, "area": "deception", "dimension": "propensity", "metricIds": [ "saferai-glm52-mask-m0", "saferai-glm52-mask-m1", "saferai-glm52-mask-m2", "saferai-glm52-mask-m3", "saferai-glm52-mask-m4", "saferai-glm52-mask-m5" ], "variantIds": [ "saferai-glm52-mask" ], "evidenceLinkIds": [ "link-saferai-glm52-mask-glm-5-2", "link-saferai-glm52-mask-claude-opus-4-7", "link-saferai-glm52-mask-gpt-5-5" ], "connection": "points", "description": { "en": "Source-specific results by model release. Protocol and evaluator scope remain explicit.", "zh": "按模型发布时间展示来源特定结果，保留协议与评测方范围。" }, "limitations": { "en": "A controlled benchmark does not establish stable intent or real-world deception prevalence.", "zh": "受控基准不证明稳定意图或现实环境中的欺骗发生率。" }, "sourceIds": [ "saferai-glm52-source" ] }, { "id": "caisi-deepseek-v4-ctf-archive-diamond", "title": { "en": "CTF-Archive-Diamond", "zh": "CTF-Archive-Diamond" }, "area": "cybersecurity", "dimension": "capability", "metricIds": [ "caisi-deepseek-v4-ctf-archive-diamond-m0" ], "variantIds": [ "caisi-deepseek-v4-ctf-archive-diamond" ], "evidenceLinkIds": [ "link-caisi-deepseek-v4-ctf-archive-diamond-gpt-5-5", "link-caisi-deepseek-v4-ctf-archive-diamond-gpt-5-4-mini", "link-caisi-deepseek-v4-ctf-archive-diamond-claude-opus-4-6", "link-caisi-deepseek-v4-ctf-archive-diamond-deepseek-v4-pro" ], "connection": "points", "description": { "en": "Source-specific results by model release. Protocol and evaluator scope remain explicit.", "zh": "按模型发布时间展示来源特定结果，保留协议与评测方范围。" }, "limitations": { "en": "CTF tasks are not live operational attacks; subset-imputed values are not directly observed full-set success rates.", "zh": "CTF 不是现实行动攻击；子集插补值不是全量直接观察的成功率。" }, "sourceIds": [ "caisi-deepseek-v4-source" ] } ], "scenarioAssociations": [ { "workId": "multi-agent-control", "scope": "multi-agent", "sourceId": "multi-agent-control-publication", "configuration": { "en": "FakeLab is a synthetic AI-lab monorepo. Sequential agents receive legitimate coding tasks and a shared malicious side task; state passes through committed code. The study varies sequence length, a separate planner, write permissions and monitor capability. Its main outcome requires all main tasks and the side task to succeed without any commit crossing the monitoring threshold.", "zh": "FakeLab 是模拟 AI 实验室的单体代码库。顺序运行的智能体分别接收正常编码任务和共同的恶意附加任务，通过已提交代码传递状态。研究改变序列长度、独立规划器、写权限和监测器能力；主要结果要求所有正常任务与附加任务均成功，且没有提交触发监测阈值。" }, "locator": "§2–4" }, { "workId": "narcbench", "scope": "multi-agent", "sourceId": "narcbench-publication", "configuration": { "en": "NARCBench uses four agents, with two instructed colluders in collusion runs and an all-honest control. Probes learn a deception direction from Core scenarios; five aggregation methods classify whole runs. Evaluation separates held-out Core domains, structurally different Transfer scenarios and a steganographic card-game tier, across four open-weight models.", "zh": "NARCBench 使用四个智能体：串谋实验明确指定两个串谋者，对照实验全部诚实。探针从 Core 场景学习欺骗方向，再用五种汇总方法对整次运行分类。四个开放权重模型分别在留出 Core 领域、结构不同的 Transfer 场景和隐写纸牌游戏层级接受评估。" }, "locator": "§3–4.6" }, { "workId": "opendeception", "scope": "multi-agent", "sourceId": "opendeception", "configuration": { "en": "Model in simulated social interactions", "zh": "模拟社会交互中的模型" }, "locator": "Methodology" } ], "sourceInventory": [ { "id": "anthropic-anthropic-february-2026-risk-report", "sourceId": "official-anthropic-anthropic-february-2026-risk-report", "developerId": "anthropic", "title": "Anthropic February 2026 Risk Report", "documentType": "risk-report", "publicationDate": null, "updatedDate": "2026-07-08", "models": [], "reviewLevel": "section-map-only", "sections": [ { "heading": "Autonomy and chemical/biological risk chapters", "locator": "PDF table of contents", "protocolNames": [], "disposition": "section-map-reviewed; full-risk-argument-review-pending", "notes": { "en": "Earlier corporate risk assessment with explicit May and July editorial revisions; retain later card/report context.", "zh": "早期公司级风险评估，含明确5月及7月修订；须保留后续模型卡/报告语境。" } } ], "dateEvidence": [ { "field": "updatedDate", "value": "2026-07-08", "source": "PDF front-matter changelog" } ] }, { "id": "anthropic-claude-2-system-card", "sourceId": "official-anthropic-claude-2-system-card", "developerId": "anthropic", "title": "Claude 2 System Card", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "Claude 2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "anthropic-claude-3-system-card", "sourceId": "official-anthropic-claude-3-system-card", "developerId": "anthropic", "title": "Claude 3 System Card", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "Claude 3" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "anthropic-claude-haiku-3-5-and-sonnet-3-5-new-system-card", "sourceId": "official-anthropic-claude-haiku-3-5-and-sonnet-3-5-new-system-card", "developerId": "anthropic", "title": "Claude Haiku 3.5 and Sonnet 3.5 (new) System Card", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "Claude Haiku 3.5 and Sonnet 3.5 (new)" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "anthropic-claude-sonnet-3-5-system-card", "sourceId": "official-anthropic-claude-sonnet-3-5-system-card", "developerId": "anthropic", "title": "Claude Sonnet 3.5 System Card", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "Claude Sonnet 3.5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "anthropic-claude-sonnet-3-7-system-card", "sourceId": "official-anthropic-claude-sonnet-3-7-system-card", "developerId": "anthropic", "title": "Claude Sonnet 3.7 System Card", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "Claude Sonnet 3.7" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "anthropic-claude-sonnet-4-and-opus-4-system-card", "sourceId": "official-anthropic-claude-sonnet-4-and-opus-4-system-card", "developerId": "anthropic", "title": "Claude Sonnet 4 and Opus 4 System Card", "documentType": "system-card", "publicationDate": "2025-05-04", "updatedDate": null, "models": [ "Claude Sonnet 4 and Opus 4" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-05-04", "source": "PDF title page" } ] }, { "id": "anthropic-claude-opus-4-1-system-card", "sourceId": "official-anthropic-claude-opus-4-1-system-card", "developerId": "anthropic", "title": "Claude Opus 4.1 System Card", "documentType": "system-card", "publicationDate": "2025-08-01", "updatedDate": "2025-09-15", "models": [ "Claude Opus 4.1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-08-01", "source": "PDF title page" }, { "field": "updatedDate", "value": "2025-09-15", "source": "PDF front-matter changelog" } ] }, { "id": "anthropic-claude-sonnet-4-5-system-card", "sourceId": "official-anthropic-claude-sonnet-4-5-system-card", "developerId": "anthropic", "title": "Claude Sonnet 4.5 System Card", "documentType": "system-card", "publicationDate": "2025-09-05", "updatedDate": "2025-12-03", "models": [ "Claude Sonnet 4.5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-09-05", "source": "PDF title page" }, { "field": "updatedDate", "value": "2025-12-03", "source": "PDF front-matter changelog" } ] }, { "id": "anthropic-claude-haiku-4-5-system-card", "sourceId": "official-anthropic-claude-haiku-4-5-system-card", "developerId": "anthropic", "title": "Claude Haiku 4.5 System Card", "documentType": "system-card", "publicationDate": "2025-10-05", "updatedDate": null, "models": [ "Claude Haiku 4.5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-10-05", "source": "PDF title page" } ] }, { "id": "anthropic-claude-opus-4-5-system-card", "sourceId": "official-anthropic-claude-opus-4-5-system-card", "developerId": "anthropic", "title": "Claude Opus 4.5 System Card", "documentType": "system-card", "publicationDate": "2025-11-05", "updatedDate": "2025-12-05", "models": [ "Claude Opus 4.5" ], "reviewLevel": "section-map-only", "sections": [ { "heading": "RSP, cyber and alignment sections", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "protocolNames": [ "Cybench", "CyberGym", "SHADE-Arena", "Petri" ], "disposition": "section-map-reviewed; result-level-comparison-pending", "notes": { "en": "Historical comparator source located. Newer card summaries and external evaluations must preserve the tested model version, harness and safeguard configuration.", "zh": "已定位历史比较来源；新卡摘要与外部评估须保留模型版本、框架及防护配置。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2025-11-05", "source": "PDF title page" }, { "field": "updatedDate", "value": "2025-12-05", "source": "PDF front-matter changelog" } ] }, { "id": "anthropic-claude-opus-4-6-system-card-retry", "sourceId": "official-anthropic-claude-opus-4-6-system-card-retry", "developerId": "anthropic", "title": "Claude Opus 4.6 System Card", "documentType": "system-card", "publicationDate": "2026-02-06", "updatedDate": "2026-03-06", "models": [ "Claude Opus 4.6" ], "reviewLevel": "section-map-only", "sections": [ { "heading": "RSP, agentic safety and alignment assessment", "locator": "PDF contents → named RSP, agentic safety and alignment sections", "protocolNames": [ "Cybench", "CyberGym", "MASK", "SHADE-Arena" ], "disposition": "section-map-reviewed; result-level-comparison-pending", "notes": { "en": "Historical comparator for Neo, SaferAI and CAISI; safety-card configuration must be checked before importing a result.", "zh": "Neo、SaferAI及CAISI的历史比较模型；导入结果前须核对安全卡配置。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-02-06", "source": "PDF title page" }, { "field": "updatedDate", "value": "2026-03-06", "source": "PDF front-matter changelog" } ] }, { "id": "anthropic-claude-opus-4-6-sabotage-risk-report", "sourceId": "official-anthropic-claude-opus-4-6-sabotage-risk-report", "developerId": "anthropic", "title": "Claude Opus 4.6 Sabotage Risk Report", "documentType": "risk-report", "publicationDate": "2026-02-10", "updatedDate": null, "models": [ "Claude Opus 4.6" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "4 Capabilities and behaviors; 5 Risk mitigations; 6 Overall risk", "locator": "PDF §§4–6", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Developer argues sabotage risk is very low but non-negligible using alignment evidence and controls. Intentional human misuse is outside this report’s principal threat model.", "zh": "开发者结合对齐证据与控制论证破坏风险很低但非零；蓄意人为滥用并非本报告主要威胁模型。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-02-10", "source": "Official RSP page, February 10 announcement; corroborated by Opus 4.6 card changelog." } ] }, { "id": "anthropic-claude-sonnet-4-6-system-card", "sourceId": "official-anthropic-claude-sonnet-4-6-system-card", "developerId": "anthropic", "title": "Claude Sonnet 4.6 System Card", "documentType": "system-card", "publicationDate": "2026-02-17", "updatedDate": "2026-03-06", "models": [ "Claude Sonnet 4.6" ], "reviewLevel": "section-map-only", "sections": [ { "heading": "RSP, cyber and alignment sections", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "protocolNames": [ "Cybench", "CyberGym", "SHADE-Arena", "Petri" ], "disposition": "section-map-reviewed; result-level-comparison-pending", "notes": { "en": "Historical comparator source located. Newer card summaries and external evaluations must preserve the tested model version, harness and safeguard configuration.", "zh": "已定位历史比较来源；新卡摘要与外部评估须保留模型版本、框架及防护配置。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-02-17", "source": "PDF title page" }, { "field": "updatedDate", "value": "2026-03-06", "source": "PDF front-matter changelog" } ] }, { "id": "anthropic-claude-mythos-preview-risk-report", "sourceId": "official-anthropic-claude-mythos-preview-risk-report", "developerId": "anthropic", "title": "Alignment Risk Update: Claude Mythos Preview", "documentType": "risk-report", "publicationDate": "2026-04-07", "updatedDate": "2026-04-10", "models": [ "Claude Mythos Preview" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "5 Evidence; 6 Risk mitigations; 10 Overall assessment", "locator": "PDF contents → Evidence, Risk mitigations, Overall assessment", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "A model-specific alignment risk update; subsequent cards and later corporate risk reports supersede parts of this assessment.", "zh": "模型特定的对齐风险更新；后续模型卡与公司风险报告更新了部分评估。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-04-07", "source": "PDF title page" }, { "field": "updatedDate", "value": "2026-04-10", "source": "PDF changelog April 10th, 2026" } ] }, { "id": "anthropic-mythos-preview-system-card", "sourceId": "official-anthropic-mythos-preview-system-card", "developerId": "anthropic", "title": "Mythos Preview System Card", "documentType": "system-card", "publicationDate": "2026-04-07", "updatedDate": "2026-04-14", "models": [ "Mythos Preview" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-04-07", "source": "PDF title page" }, { "field": "updatedDate", "value": "2026-04-14", "source": "PDF front-matter changelog" } ] }, { "id": "anthropic-claude-opus-4-7-system-card", "sourceId": "official-anthropic-claude-opus-4-7-system-card", "developerId": "anthropic", "title": "Claude Opus 4.7 System Card", "documentType": "system-card", "publicationDate": "2026-04-16", "updatedDate": null, "models": [ "Claude Opus 4.7" ], "reviewLevel": "section-map-only", "sections": [ { "heading": "RSP, cyber and alignment sections", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "protocolNames": [ "Cybench", "CyberGym", "MASK", "SHADE-Arena", "Petri" ], "disposition": "section-map-reviewed; result-level-comparison-pending", "notes": { "en": "Historical comparator source located. Newer card summaries and external evaluations must preserve the tested model version, harness and safeguard configuration.", "zh": "已定位历史比较来源；新卡摘要与外部评估须保留模型版本、框架及防护配置。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-04-16", "source": "PDF title page" } ] }, { "id": "anthropic-claude-opus-4-8-system-card", "sourceId": "official-anthropic-claude-opus-4-8-system-card", "developerId": "anthropic", "title": "Claude Opus 4.8 System Card", "documentType": "system-card", "publicationDate": "2026-05-28", "updatedDate": "2026-06-17", "models": [ "Claude Opus 4.8" ], "reviewLevel": "section-map-only", "sections": [ { "heading": "RSP, cyber and alignment sections", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "protocolNames": [ "Cybench", "CyberGym", "ExploitBench", "MASK", "SHADE-Arena", "Petri" ], "disposition": "section-map-reviewed; result-level-comparison-pending", "notes": { "en": "Historical comparator source located. Newer card summaries and external evaluations must preserve the tested model version, harness and safeguard configuration.", "zh": "已定位历史比较来源；新卡摘要与外部评估须保留模型版本、框架及防护配置。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-05-28", "source": "PDF title page" }, { "field": "updatedDate", "value": "2026-06-17", "source": "PDF front-matter changelog" } ] }, { "id": "anthropic-claude-fable-5-and-mythos-5-system-card", "sourceId": "official-anthropic-claude-fable-5-and-mythos-5-system-card", "developerId": "anthropic", "title": "Claude Fable 5 and Mythos 5 System Card", "documentType": "system-card", "publicationDate": "2026-06-09", "updatedDate": "2026-06-25", "models": [ "Claude Fable 5 and Mythos 5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-06-09", "source": "PDF title page" }, { "field": "updatedDate", "value": "2026-06-25", "source": "PDF front-matter changelog" } ] }, { "id": "anthropic-claude-sonnet-5-system-card", "sourceId": "official-anthropic-claude-sonnet-5-system-card", "developerId": "anthropic", "title": "Claude Sonnet 5 System Card", "documentType": "system-card", "publicationDate": "2026-06-30", "updatedDate": "2026-07-10", "models": [ "Claude Sonnet 5" ], "reviewLevel": "section-map-only", "sections": [ { "heading": "RSP, cyber and alignment sections", "locator": "PDF table of contents → named RSP / cyber / alignment sections", "protocolNames": [ "CyberGym", "ExploitBench", "MASK", "SHADE-Arena" ], "disposition": "section-map-reviewed; result-level-comparison-pending", "notes": { "en": "Historical comparator source located. Newer card summaries and external evaluations must preserve the tested model version, harness and safeguard configuration.", "zh": "已定位历史比较来源；新卡摘要与外部评估须保留模型版本、框架及防护配置。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-06-30", "source": "PDF title page" }, { "field": "updatedDate", "value": "2026-07-10", "source": "PDF front-matter changelog" } ] }, { "id": "anthropic-claude-opus-5-system-card", "sourceId": "official-anthropic-claude-opus-5-system-card", "developerId": "anthropic", "title": "Claude Opus 5 System Card", "documentType": "system-card", "publicationDate": "2026-07-24", "updatedDate": "2026-08-19", "models": [ "Claude Opus 5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-07-24", "source": "PDF title page" }, { "field": "updatedDate", "value": "2026-08-19", "source": "PDF front-matter changelog" } ] }, { "id": "anthropic-anthropic-august-2026-risk-report", "sourceId": "official-anthropic-anthropic-august-2026-risk-report", "developerId": "anthropic", "title": "Anthropic August 2026 Risk Report", "documentType": "risk-report", "publicationDate": "2026-08-14", "updatedDate": null, "models": [ "Claude model family", "internal models discussed in redacted report" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "2 Autonomy threat model 1: Misalignment in high-stakes settings", "locator": "PDF printed pp.17–88", "protocolNames": [ "SHADE-Arena", "LinuxArena" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Structured claims, monitoring, mitigation assumptions and limitations support a low-risk developer judgment; redactions constrain independent review.", "zh": "结构化主张、监测、缓解假设与局限支撑开发者的低风险判断；删节限制独立复核。" } }, { "heading": "3 Autonomy threat model 2: Risks from automated R&D", "locator": "PDF printed pp.93–113", "protocolNames": [ "CoBench", "AECI" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Separates researcher substitution, observed acceleration and forward-looking risks; capability proxies do not establish a measured economy-wide acceleration.", "zh": "区分替代研究者、已观察加速与前瞻风险；能力代理指标不能建立全经济加速结论。" } }, { "heading": "4 Chemical and biological weapons production", "locator": "PDF printed pp.115–155", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Threat models, evidence weighting and mitigations reviewed at a high level only; no hazardous task details or quantitative results retained.", "zh": "仅高层审阅威胁模型、证据权重与防护；不保留危险任务细节或数值。" } }, { "heading": "5.2 Safety process failures", "locator": "PDF printed pp.163–168", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Developer discusses safety-process failures separately from capability evaluations; these are investigation/governance evidence.", "zh": "开发者将安全流程失败与能力评测区分讨论，属于调查与治理证据。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-08-14", "source": "https://www.anthropic.com/responsible-scaling-policy → August 14, 2026 announcement; distinct from July 15 coverage date." } ] }, { "id": "anthropic-claude-fable-5-1-and-mythos-5-1-system-card", "sourceId": "official-anthropic-claude-fable-5-1-and-mythos-5-1-system-card", "developerId": "anthropic", "title": "Claude Fable 5.1 and Mythos 5.1 System Card", "documentType": "system-card", "publicationDate": "2026-09-01", "updatedDate": null, "models": [ "Claude Fable 5.1", "Claude Mythos 5.1" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "2 RSP evaluations", "locator": "PDF §2 (see table of contents)", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Developer threshold and mitigation assessment; intrinsic capabilities, safeguarded deployment and access configurations must stay distinct.", "zh": "开发者阈值与缓解措施评估；须区分内在能力、带防护部署和访问配置。" } }, { "heading": "3 Cyber", "locator": "PDF §3", "protocolNames": [ "ExploitBench", "OSS-Fuzz", "ExploitGym", "Firefox 147" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Cyber capability tests and safeguards have different settings; the report does not establish unrestricted real-world attack success.", "zh": "网络能力测试与防护测试配置不同；不构成现实中无限制攻击成功的证据。" } }, { "heading": "5 Agentic safety", "locator": "PDF §5", "protocolNames": [ "Indirect Prompt Injection (IPI)" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Malicious use and prompt-injection tests cover coding, browser and computer-use surfaces; internal and external tests are distinguished.", "zh": "恶意使用与提示注入测试涵盖代码、浏览器与计算机操作；保留内部和外部测试差异。" } }, { "heading": "6 Alignment assessment", "locator": "PDF §6", "protocolNames": [ "MASK", "SHADE-Arena", "LinuxArena" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Behavioral audits, honesty, oversight evasion and evaluation-awareness evidence are bounded by artificial scenarios and available reasoning traces.", "zh": "行为审计、诚实性、规避监督及评测意识证据受模拟场景与可见推理轨迹限制。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-09-01", "source": "PDF title page" } ] }, { "id": "anthropic-claude-opus-5-5-system-card", "sourceId": "official-anthropic-claude-opus-5-5-system-card", "developerId": "anthropic", "title": "Claude Opus 5.5 System Card", "documentType": "system-card", "publicationDate": "2026-09-22", "updatedDate": null, "models": [ "Claude Opus 5.5" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "2 RSP evaluations", "locator": "PDF §2 (see table of contents)", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Developer threshold and mitigation assessment; intrinsic capabilities, safeguarded deployment and access configurations must stay distinct.", "zh": "开发者阈值与缓解措施评估；须区分内在能力、带防护部署和访问配置。" } }, { "heading": "3 Cyber", "locator": "PDF §3", "protocolNames": [ "ExploitBench", "CyScenarioBench", "Binary Exploitation Benchmark", "OSS-Fuzz", "ExploitGym" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Cyber capability tests and safeguards have different settings; the report does not establish unrestricted real-world attack success.", "zh": "网络能力测试与防护测试配置不同；不构成现实中无限制攻击成功的证据。" } }, { "heading": "5 Agentic safety", "locator": "PDF §5", "protocolNames": [ "Indirect Prompt Injection (IPI)" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Malicious use and prompt-injection tests cover coding, browser and computer-use surfaces; internal and external tests are distinguished.", "zh": "恶意使用与提示注入测试涵盖代码、浏览器与计算机操作；保留内部和外部测试差异。" } }, { "heading": "6 Alignment assessment", "locator": "PDF §6", "protocolNames": [ "MASK", "SHADE-Arena", "LinuxArena" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Behavioral audits, honesty, oversight evasion and evaluation-awareness evidence are bounded by artificial scenarios and available reasoning traces.", "zh": "行为审计、诚实性、规避监督及评测意识证据受模拟场景与可见推理轨迹限制。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-09-22", "source": "PDF title page" } ] }, { "id": "anthropic-claude-sonnet-5-5-system-card", "sourceId": "official-anthropic-claude-sonnet-5-5-system-card", "developerId": "anthropic", "title": "Claude Sonnet 5.5 System Card", "documentType": "system-card", "publicationDate": "2026-09-28", "updatedDate": null, "models": [ "Claude Sonnet 5.5" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "2 RSP evaluations", "locator": "PDF §2 (see table of contents)", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Developer threshold and mitigation assessment; intrinsic capabilities, safeguarded deployment and access configurations must stay distinct.", "zh": "开发者阈值与缓解措施评估；须区分内在能力、带防护部署和访问配置。" } }, { "heading": "3 Cyber", "locator": "PDF §3", "protocolNames": [ "ExploitBench", "CyScenarioBench", "Binary Exploitation Benchmark", "OSS-Fuzz", "ExploitGym" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Cyber capability tests and safeguards have different settings; the report does not establish unrestricted real-world attack success.", "zh": "网络能力测试与防护测试配置不同；不构成现实中无限制攻击成功的证据。" } }, { "heading": "5 Agentic safety", "locator": "PDF §5", "protocolNames": [ "Indirect Prompt Injection (IPI)" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Malicious use and prompt-injection tests cover coding, browser and computer-use surfaces; internal and external tests are distinguished.", "zh": "恶意使用与提示注入测试涵盖代码、浏览器与计算机操作；保留内部和外部测试差异。" } }, { "heading": "6 Alignment assessment", "locator": "PDF §6", "protocolNames": [ "MASK", "SHADE-Arena", "LinuxArena" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Behavioral audits, honesty, oversight evasion and evaluation-awareness evidence are bounded by artificial scenarios and available reasoning traces.", "zh": "行为审计、诚实性、规避监督及评测意识证据受模拟场景与可见推理轨迹限制。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-09-28", "source": "PDF title page" } ] }, { "id": "anthropic-claude-haiku-5-5-system-card", "sourceId": "official-anthropic-claude-haiku-5-5-system-card", "developerId": "anthropic", "title": "Claude Haiku 5.5 System Card", "documentType": "system-card", "publicationDate": "2026-10-07", "updatedDate": null, "models": [ "Claude Haiku 5.5" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "2 RSP evaluations", "locator": "PDF §2 (see table of contents)", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Developer threshold and mitigation assessment; intrinsic capabilities, safeguarded deployment and access configurations must stay distinct.", "zh": "开发者阈值与缓解措施评估；须区分内在能力、带防护部署和访问配置。" } }, { "heading": "3 Cyber", "locator": "PDF §3", "protocolNames": [ "ExploitBench", "CyScenarioBench", "Binary Exploitation Benchmark", "OSS-Fuzz", "ExploitGym" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Cyber capability tests and safeguards have different settings; the report does not establish unrestricted real-world attack success.", "zh": "网络能力测试与防护测试配置不同；不构成现实中无限制攻击成功的证据。" } }, { "heading": "5 Agentic safety", "locator": "PDF §5", "protocolNames": [ "Indirect Prompt Injection (IPI)" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Malicious use and prompt-injection tests cover coding, browser and computer-use surfaces; internal and external tests are distinguished.", "zh": "恶意使用与提示注入测试涵盖代码、浏览器与计算机操作；保留内部和外部测试差异。" } }, { "heading": "6 Alignment assessment", "locator": "PDF §6", "protocolNames": [ "MASK", "SHADE-Arena", "LinuxArena" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Behavioral audits, honesty, oversight evasion and evaluation-awareness evidence are bounded by artificial scenarios and available reasoning traces.", "zh": "行为审计、诚实性、规避监督及评测意识证据受模拟场景与可见推理轨迹限制。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-10-07", "source": "PDF title page" } ] }, { "id": "deepseek-deepseek-llm-scaling-open-source-language-models-with-longtermism", "sourceId": "official-deepseek-deepseek-llm-scaling-open-source-language-models-with-longtermism", "developerId": "deepseek", "title": "DeepSeek LLM: Scaling Open-Source Language Models with Longtermism", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek LLM" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2311.18743 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "deepseek-deepseek-r1-model-card", "sourceId": "official-deepseek-deepseek-r1-model-card", "developerId": "deepseek", "title": "DeepSeek-R1 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek-R1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "deepseek-deepseek-r1-0528-model-card", "sourceId": "official-deepseek-deepseek-r1-0528-model-card", "developerId": "deepseek", "title": "DeepSeek-R1-0528 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek-R1-0528" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "deepseek-deepseek-r1-incentivizing-reasoning-capability-in-llms-via-reinforcement-learning", "sourceId": "official-deepseek-deepseek-r1-incentivizing-reasoning-capability-in-llms-via-reinforcement-learning", "developerId": "deepseek", "title": "DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek-R1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2501.12948 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "deepseek-deepseek-v2-model-card", "sourceId": "official-deepseek-deepseek-v2-model-card", "developerId": "deepseek", "title": "DeepSeek-V2 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek-V2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "deepseek-deepseek-v2-a-strong-economical-and-efficient-mixture-of-experts-language-model", "sourceId": "official-deepseek-deepseek-v2-a-strong-economical-and-efficient-mixture-of-experts-language-model", "developerId": "deepseek", "title": "DeepSeek-V2: A Strong, Economical, and Efficient Mixture-of-Experts Language Model", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek-V2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "deepseek-deepseek-v3-model-card", "sourceId": "official-deepseek-deepseek-v3-model-card", "developerId": "deepseek", "title": "DeepSeek-V3 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek-V3" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "deepseek-deepseek-v3-technical-report", "sourceId": "official-deepseek-deepseek-v3-technical-report", "developerId": "deepseek", "title": "DeepSeek-V3 Technical Report", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek-V3" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2412.19437 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "deepseek-deepseek-v3-1-model-card", "sourceId": "official-deepseek-deepseek-v3-1-model-card", "developerId": "deepseek", "title": "DeepSeek-V3.1 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek-V3.1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "deepseek-deepseek-v3-2-model-card", "sourceId": "official-deepseek-deepseek-v3-2-model-card", "developerId": "deepseek", "title": "DeepSeek-V3.2 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek-V3.2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "deepseek-deepseek-v3-2-exp-model-card", "sourceId": "official-deepseek-deepseek-v3-2-exp-model-card", "developerId": "deepseek", "title": "DeepSeek-V3.2-Exp Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek-V3.2-Exp" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "deepseek-deepseek-v3-2-efficient-reasoning-agentic-ai", "sourceId": "official-deepseek-deepseek-v3-2-efficient-reasoning-agentic-ai", "developerId": "deepseek", "title": "DeepSeek-V3.2: Efficient Reasoning & Agentic AI", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek-V3.2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "deepseek-deepseek-v4-flash-model-card", "sourceId": "official-deepseek-deepseek-v4-flash-model-card", "developerId": "deepseek", "title": "DeepSeek-V4-Flash Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek-V4-Flash" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "deepseek-deepseek-v4-pro-model-card", "sourceId": "official-deepseek-deepseek-v4-pro-model-card", "developerId": "deepseek", "title": "DeepSeek-V4-Pro Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek-V4-Pro" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Evaluation Results", "locator": "Original source → Evaluation Results", "protocolNames": [], "disposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established", "notes": { "en": "Capability table and deployment guidance do not substitute for CAISI or Neo independent risk evaluations.", "zh": "能力表与部署说明不能替代CAISI或Neo独立风险评估。" } } ], "dateEvidence": [] }, { "id": "deepseek-deepseek-v4-towards-highly-efficient-million-token-context-intelligence", "sourceId": "official-deepseek-deepseek-v4-towards-highly-efficient-million-token-context-intelligence", "developerId": "deepseek", "title": "DeepSeek-V4: Towards Highly Efficient Million-Token Context Intelligence", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "DeepSeek-V4-Pro", "DeepSeek-V4-Flash" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "5 Post-Training; 6 Conclusion, Limitations, and Future Directions", "locator": "Original source → 5 Post-Training; 6 Conclusion, Limitations, and Future Directions", "protocolNames": [], "disposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established", "notes": { "en": "Architecture, training and general capabilities dominate; limitations include task/reasoning shortcomings. No dedicated frontier-risk assessment located.", "zh": "重点为架构、训练与一般能力；局限包含任务与推理不足，未发现独立前沿风险评估。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2606.19348 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "deepseek-deepseek-llm-67b-chat-model-card", "sourceId": "official-deepseek-deepseek-llm-67b-chat-model-card", "developerId": "deepseek", "title": "deepseek-llm-67b-chat Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "deepseek-llm-67b-chat" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "google-deepmind-codegemma-model-card", "sourceId": "official-google-deepmind-codegemma-model-card", "developerId": "deepmind", "title": "CodeGemma Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-02-25", "models": [ "CodeGemma" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-02-25", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-diffusiongemma-model-card", "sourceId": "official-google-deepmind-diffusiongemma-model-card", "developerId": "deepmind", "title": "DiffusionGemma Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2026-06-10", "models": [ "DiffusionGemma" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-06-10", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-embeddinggemma-2-model-card", "sourceId": "official-google-deepmind-embeddinggemma-2-model-card", "developerId": "deepmind", "title": "EmbeddingGemma 2 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2026-10-06", "models": [ "EmbeddingGemma 2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-10-06", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-embeddinggemma-model-card", "sourceId": "official-google-deepmind-embeddinggemma-model-card", "developerId": "deepmind", "title": "EmbeddingGemma Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-09-25", "models": [ "EmbeddingGemma" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-09-25", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-functiongemma-model-card", "sourceId": "official-google-deepmind-functiongemma-model-card", "developerId": "deepmind", "title": "FunctionGemma Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2026-01-14", "models": [ "FunctionGemma" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-01-14", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemini-1-0-model-card", "sourceId": "official-google-deepmind-gemini-1-0-model-card", "developerId": "deepmind", "title": "Gemini 1.0 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Gemini 1.0" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "google-deepmind-gemini-1-5-model-card", "sourceId": "official-google-deepmind-gemini-1-5-model-card", "developerId": "deepmind", "title": "Gemini 1.5 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Gemini 1.5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "google-deepmind-gemini-2-0-flash-model-card", "sourceId": "official-google-deepmind-gemini-2-0-flash-model-card", "developerId": "deepmind", "title": "Gemini 2.0 Flash Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-04-15", "models": [ "Gemini 2.0 Flash" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-04-15", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemini-2-0-flash-lite-model-card", "sourceId": "official-google-deepmind-gemini-2-0-flash-lite-model-card", "developerId": "deepmind", "title": "Gemini 2.0 Flash-Lite Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-04-10", "models": [ "Gemini 2.0 Flash-Lite" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-04-10", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemini-2-5-computer-use-model-card", "sourceId": "official-google-deepmind-gemini-2-5-computer-use-model-card", "developerId": "deepmind", "title": "Gemini 2.5 Computer Use Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-10-07", "models": [ "Gemini 2.5 Computer Use" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-10-07", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemini-2-5-deep-think-model-card", "sourceId": "official-google-deepmind-gemini-2-5-deep-think-model-card", "developerId": "deepmind", "title": "Gemini 2.5 Deep Think Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-08-01", "models": [ "Gemini 2.5 Deep Think" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-08-01", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemini-2-5-flash-and-gemini-2-5-flash-image-model-card", "sourceId": "official-google-deepmind-gemini-2-5-flash-and-gemini-2-5-flash-image-model-card", "developerId": "deepmind", "title": "Gemini 2.5 Flash and Gemini 2.5 Flash Image Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-09-26", "models": [ "Gemini 2.5 Flash and Gemini 2.5 Flash Image" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-09-26", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemini-2-5-flash-lite-model-card", "sourceId": "official-google-deepmind-gemini-2-5-flash-lite-model-card", "developerId": "deepmind", "title": "Gemini 2.5 Flash-Lite Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-09-26", "models": [ "Gemini 2.5 Flash-Lite" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-09-26", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemini-2-5-pro-model-card", "sourceId": "official-google-deepmind-gemini-2-5-pro-model-card", "developerId": "deepmind", "title": "Gemini 2.5 Pro Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-06-27", "models": [ "Gemini 2.5 Pro" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-06-27", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemini-2-5-pushing-the-frontier-with-advanced-reasoning-multimodality-long-context-and-next-generation-ag", "sourceId": "official-google-deepmind-gemini-2-5-pushing-the-frontier-with-advanced-reasoning-multimodality-long-context-and-next-generation-ag", "developerId": "deepmind", "title": "Gemini 2.5: Pushing the Frontier with Advanced Reasoning, Multimodality, Long Context, and Next Generation Agentic Capabilities", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "Gemini 2.5 Pro", "Gemini 2.5 Flash" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "google-deepmind-gemini-3-flash-model-card", "sourceId": "official-google-deepmind-gemini-3-flash-model-card", "developerId": "deepmind", "title": "Gemini 3 Flash Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-12-17", "models": [ "Gemini 3 Flash" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-12-17", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemini-3-pro-image-model-card", "sourceId": "official-google-deepmind-gemini-3-pro-image-model-card", "developerId": "deepmind", "title": "Gemini 3 Pro Image Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-11-20", "models": [ "Gemini 3 Pro Image" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-11-20", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemini-3-pro-model-card", "sourceId": "official-google-deepmind-gemini-3-pro-model-card", "developerId": "deepmind", "title": "Gemini 3 Pro Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-11-18", "models": [ "Gemini 3 Pro" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-11-18", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemini-4-argon-safety-security-report", "sourceId": "official-google-deepmind-gemini-4-argon-safety-security-report", "developerId": "deepmind", "title": "Gemini 4 Argon Safety & Security Report", "documentType": "safety-security-report", "publicationDate": null, "updatedDate": null, "models": [ "Gemini 4 Argon" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "3.2 Misuse risk assessment", "locator": "PDF printed pp.21–53", "protocolNames": [ "ExploitBench" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Separates inherent capability from residual risk and uses internal and independent tests; no operational biology details or numerical scores are transcribed.", "zh": "区分内在能力与剩余风险，结合内部及独立测试；未转录生物操作细节或数值。" } }, { "heading": "3.3 Misuse Safeguards", "locator": "PDF printed pp.55–62", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Coverage and adversarial robustness are assessed separately; safeguards are part of the deployment risk argument.", "zh": "分别评估覆盖度与对抗稳健性；防护措施是部署风险论证的组成部分。" } }, { "heading": "3.4 Loss of Control", "locator": "PDF printed pp.63–155", "protocolNames": [ "Gram", "LinuxArena" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Holistic safety argument combines capabilities, propensities, monitoring and security assumptions; conclusions depend on those stated assumptions.", "zh": "综合安全论证结合能力、倾向、监测与安全假设；结论依赖所述假设。" } }, { "heading": "3.4.5 Chain-of-Thought Transparency Assessment", "locator": "PDF printed pp.112–155", "protocolNames": [ "CoT controllability", "monitor evasion" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Developer reports more difficult monitorability than prior models; complex covert-task results are bounded by task difficulty and monitor strength.", "zh": "开发者称可监测性较前代更困难；复杂隐蔽任务结论受任务难度与监测器强度限制。" } }, { "heading": "3.5 Evaluation Awareness and Behavior Effects", "locator": "PDF printed pp.156–158", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Awareness can change compliance and local grader behavior. Audits found no confirmed strategic underperformance; this does not rule out unspoken awareness.", "zh": "评测意识可改变合规及局部评分行为；审计未确认战略性低报能力，但不排除未表述的意识。" } }, { "heading": "6 Guarding users against harm from third parties", "locator": "PDF §6", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Prompt-injection resistance is a separate model/system robustness finding.", "zh": "提示注入抵抗力属于独立的模型/系统稳健性结论。" } } ], "dateEvidence": [] }, { "id": "google-deepmind-gemini-robotics-1-5-model-card", "sourceId": "official-google-deepmind-gemini-robotics-1-5-model-card", "developerId": "deepmind", "title": "Gemini Robotics 1.5 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-09-25", "models": [ "Gemini Robotics 1.5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-09-25", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemini-robotics-on-device-model-card", "sourceId": "official-google-deepmind-gemini-robotics-on-device-model-card", "developerId": "deepmind", "title": "Gemini Robotics On-Device Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-07-01", "models": [ "Gemini Robotics On-Device" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-07-01", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemma-1-model-card", "sourceId": "official-google-deepmind-gemma-1-model-card", "developerId": "deepmind", "title": "Gemma 1 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-02-25", "models": [ "Gemma 1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-02-25", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemma-2-model-card", "sourceId": "official-google-deepmind-gemma-2-model-card", "developerId": "deepmind", "title": "Gemma 2 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-02-25", "models": [ "Gemma 2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-02-25", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemma-3-model-card", "sourceId": "official-google-deepmind-gemma-3-model-card", "developerId": "deepmind", "title": "Gemma 3 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-08-14", "models": [ "Gemma 3" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-08-14", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemma-3n-model-card", "sourceId": "official-google-deepmind-gemma-3n-model-card", "developerId": "deepmind", "title": "Gemma 3n Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-06-17", "models": [ "Gemma 3n" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-06-17", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-gemma-4-model-card", "sourceId": "official-google-deepmind-gemma-4-model-card", "developerId": "deepmind", "title": "Gemma 4 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2026-04-02", "models": [ "Gemma 4" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-04-02", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-imagen-4-model-card", "sourceId": "official-google-deepmind-imagen-4-model-card", "developerId": "deepmind", "title": "Imagen 4 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-05-20", "models": [ "Imagen 4" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-05-20", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-paligemma-1-model-card", "sourceId": "official-google-deepmind-paligemma-1-model-card", "developerId": "deepmind", "title": "PaliGemma 1 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-02-25", "models": [ "PaliGemma 1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-02-25", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-paligemma-2-model-card", "sourceId": "official-google-deepmind-paligemma-2-model-card", "developerId": "deepmind", "title": "PaliGemma 2 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-02-25", "models": [ "PaliGemma 2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-02-25", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-recurrentgemma-model-card", "sourceId": "official-google-deepmind-recurrentgemma-model-card", "developerId": "deepmind", "title": "RecurrentGemma Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-02-25", "models": [ "RecurrentGemma" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-02-25", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-shieldgemma-1-model-card", "sourceId": "official-google-deepmind-shieldgemma-1-model-card", "developerId": "deepmind", "title": "ShieldGemma 1 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-02-25", "models": [ "ShieldGemma 1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-02-25", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-shieldgemma-2-model-card", "sourceId": "official-google-deepmind-shieldgemma-2-model-card", "developerId": "deepmind", "title": "ShieldGemma 2 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2025-06-17", "models": [ "ShieldGemma 2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2025-06-17", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-veo-3-model-card", "sourceId": "official-google-deepmind-veo-3-model-card", "developerId": "deepmind", "title": "Veo 3 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2026-01-13", "models": [ "Veo 3" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-01-13", "source": "Official model-card index marks this date Updated." } ] }, { "id": "google-deepmind-lyria-3-model-card", "sourceId": "official-google-deepmind-lyria-3-model-card", "developerId": "deepmind", "title": "Lyria 3 Model Card", "documentType": "model-card", "publicationDate": "2026-02-18", "updatedDate": "2026-02-18", "models": [ "Lyria 3" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-02-18", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-02-18", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-3-1-pro-model-card", "sourceId": "official-google-deepmind-gemini-3-1-pro-model-card", "developerId": "deepmind", "title": "Gemini 3.1 Pro Model Card", "documentType": "model-card", "publicationDate": "2026-02-19", "updatedDate": "2026-02-19", "models": [ "Gemini 3.1 Pro" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-02-19", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-02-19", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-3-1-flash-image-model-card", "sourceId": "official-google-deepmind-gemini-3-1-flash-image-model-card", "developerId": "deepmind", "title": "Gemini 3.1 Flash Image Model Card", "documentType": "model-card", "publicationDate": "2026-02-26", "updatedDate": "2026-02-26", "models": [ "Gemini 3.1 Flash Image" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-02-26", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-02-26", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-3-1-flash-lite-model-card", "sourceId": "official-google-deepmind-gemini-3-1-flash-lite-model-card", "developerId": "deepmind", "title": "Gemini 3.1 Flash-Lite Model Card", "documentType": "model-card", "publicationDate": "2026-03-03", "updatedDate": "2026-03-03", "models": [ "Gemini 3.1 Flash-Lite" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-03-03", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-03-03", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-3-1-flash-audio-flash-live-tts-model-card", "sourceId": "official-google-deepmind-gemini-3-1-flash-audio-flash-live-tts-model-card", "developerId": "deepmind", "title": "Gemini 3.1 Flash Audio (Flash Live, TTS) Model Card", "documentType": "model-card", "publicationDate": "2026-03-26", "updatedDate": "2026-04-15", "models": [ "Gemini 3.1 Flash Audio (Flash Live, TTS)" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-04-15", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-03-26", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-veo-3-1-lite-model-card", "sourceId": "official-google-deepmind-veo-3-1-lite-model-card", "developerId": "deepmind", "title": "Veo 3.1 Lite Model Card", "documentType": "model-card", "publicationDate": "2026-04-08", "updatedDate": "2026-04-08", "models": [ "Veo 3.1 Lite" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-04-08", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-04-08", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-robotics-er-1-6-model-card", "sourceId": "official-google-deepmind-gemini-robotics-er-1-6-model-card", "developerId": "deepmind", "title": "Gemini Robotics-ER 1.6 Model Card", "documentType": "model-card", "publicationDate": "2026-04-20", "updatedDate": "2026-04-20", "models": [ "Gemini Robotics-ER 1.6" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-04-20", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-04-20", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-3-5-flash-model-card", "sourceId": "official-google-deepmind-gemini-3-5-flash-model-card", "developerId": "deepmind", "title": "Gemini 3.5 Flash Model Card", "documentType": "model-card", "publicationDate": "2026-05-19", "updatedDate": "2026-05-19", "models": [ "Gemini 3.5 Flash" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-05-19", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-05-19", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-3-1-flash-lite-image-model-card", "sourceId": "official-google-deepmind-gemini-3-1-flash-lite-image-model-card", "developerId": "deepmind", "title": "Gemini 3.1 Flash-Lite Image Model Card", "documentType": "model-card", "publicationDate": "2026-06-30", "updatedDate": "2026-06-30", "models": [ "Gemini 3.1 Flash-Lite Image" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-06-30", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-06-30", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-3-5-flash-lite-model-card", "sourceId": "official-google-deepmind-gemini-3-5-flash-lite-model-card", "developerId": "deepmind", "title": "Gemini 3.5 Flash-Lite Model Card", "documentType": "model-card", "publicationDate": "2026-07-21", "updatedDate": "2026-07-21", "models": [ "Gemini 3.5 Flash-Lite" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-07-21", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-07-21", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-3-6-flash-model-card", "sourceId": "official-google-deepmind-gemini-3-6-flash-model-card", "developerId": "deepmind", "title": "Gemini 3.6 Flash Model Card", "documentType": "model-card", "publicationDate": "2026-07-21", "updatedDate": "2026-07-21", "models": [ "Gemini 3.6 Flash" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-07-21", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-07-21", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-lyria-3-5-model-card", "sourceId": "official-google-deepmind-lyria-3-5-model-card", "developerId": "deepmind", "title": "Lyria 3.5 Model Card", "documentType": "model-card", "publicationDate": "2026-07-29", "updatedDate": "2026-07-29", "models": [ "Lyria 3.5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-07-29", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-07-29", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-robotics-on-device-2-model-card", "sourceId": "official-google-deepmind-gemini-robotics-on-device-2-model-card", "developerId": "deepmind", "title": "Gemini Robotics On-Device 2 Model Card", "documentType": "model-card", "publicationDate": "2026-07-30", "updatedDate": "2026-07-30", "models": [ "Gemini Robotics On-Device 2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-07-30", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-07-30", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-robotics-er-2-model-card", "sourceId": "official-google-deepmind-gemini-robotics-er-2-model-card", "developerId": "deepmind", "title": "Gemini Robotics-ER 2 Model Card", "documentType": "model-card", "publicationDate": "2026-07-30", "updatedDate": "2026-07-30", "models": [ "Gemini Robotics-ER 2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-07-30", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-07-30", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-3-7-flash-model-card", "sourceId": "official-google-deepmind-gemini-3-7-flash-model-card", "developerId": "deepmind", "title": "Gemini 3.7 Flash Model Card", "documentType": "model-card", "publicationDate": "2026-08-13", "updatedDate": "2026-08-13", "models": [ "Gemini 3.7 Flash" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-08-13", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-08-13", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-3-5-audio-live-translate-transcribe-transcribe-live-model-card", "sourceId": "official-google-deepmind-gemini-3-5-audio-live-translate-transcribe-transcribe-live-model-card", "developerId": "deepmind", "title": "Gemini 3.5 Audio (Live Translate, Transcribe, Transcribe Live) Model Card", "documentType": "model-card", "publicationDate": "2026-08-26", "updatedDate": "2026-08-26", "models": [ "Gemini 3.5 Audio (Live Translate, Transcribe, Transcribe Live)" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-08-26", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-08-26", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-omni-flash-model-card", "sourceId": "official-google-deepmind-gemini-omni-flash-model-card", "developerId": "deepmind", "title": "Gemini Omni Flash Model Card", "documentType": "model-card", "publicationDate": "2026-08-27", "updatedDate": "2026-08-27", "models": [ "Gemini Omni Flash" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-08-27", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-08-27", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-3-8-flash-model-card", "sourceId": "official-google-deepmind-gemini-3-8-flash-model-card", "developerId": "deepmind", "title": "Gemini 3.8 Flash Model Card", "documentType": "model-card", "publicationDate": "2026-09-02", "updatedDate": "2026-09-02", "models": [ "Gemini 3.8 Flash" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-09-02", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-09-02", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-3-8-audio-live-live-extended-thinking-flash-tts-flash-lite-tts-model-card", "sourceId": "official-google-deepmind-gemini-3-8-audio-live-live-extended-thinking-flash-tts-flash-lite-tts-model-card", "developerId": "deepmind", "title": "Gemini 3.8 Audio (Live, Live Extended Thinking, Flash TTS, Flash-Lite TTS) Model Card", "documentType": "model-card", "publicationDate": "2026-09-15", "updatedDate": "2026-09-24", "models": [ "Gemini 3.8 Audio (Live, Live Extended Thinking, Flash TTS, Flash-Lite TTS)" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-09-24", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-09-15", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-nano-banana-2-1-model-card", "sourceId": "official-google-deepmind-nano-banana-2-1-model-card", "developerId": "deepmind", "title": "Nano Banana 2.1 Model Card", "documentType": "model-card", "publicationDate": "2026-10-06", "updatedDate": "2026-10-06", "models": [ "Nano Banana 2.1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "updatedDate", "value": "2026-10-06", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-10-06", "source": "Model-card masthead Published label" } ] }, { "id": "google-deepmind-gemini-4-argon-model-card", "sourceId": "official-google-deepmind-gemini-4-argon-model-card", "developerId": "deepmind", "title": "Gemini 4 Argon Model Card", "documentType": "model-card", "publicationDate": "2026-10-13", "updatedDate": "2026-10-13", "models": [ "Gemini 4 Argon" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Frontier Safety Assessment", "locator": "Model card → Safety & Security → Frontier Safety Assessment", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Cannot rule out cyber and biological critical capability levels or the stealth/situational-awareness tracked level; residual-risk arguments rely on mitigations.", "zh": "无法排除网络、生物关键能力级别及隐蔽性/情境意识跟踪级别；剩余风险论证依赖缓解措施。" } } ], "dateEvidence": [ { "field": "updatedDate", "value": "2026-10-13", "source": "Official model-card index marks this date Updated." }, { "field": "publicationDate", "value": "2026-10-13", "source": "Model-card masthead Published label" } ] }, { "id": "meta-llama-2-open-foundation-and-fine-tuned-chat-models", "sourceId": "official-meta-llama-2-open-foundation-and-fine-tuned-chat-models", "developerId": "meta", "title": "Llama 2: Open Foundation and Fine-Tuned Chat Models", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "Llama 2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2307.09288 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "meta-llama-2-70b-chat-hf-model-card", "sourceId": "official-meta-llama-2-70b-chat-hf-model-card", "developerId": "meta", "title": "Llama-2-70b-chat-hf Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Llama-2-70b-chat-hf" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "meta-llama-3-1-405b-instruct-model-card", "sourceId": "official-meta-llama-3-1-405b-instruct-model-card", "developerId": "meta", "title": "Llama-3.1-405B-Instruct Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Llama-3.1-405B-Instruct" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "meta-llama-3-2-90b-vision-instruct-model-card", "sourceId": "official-meta-llama-3-2-90b-vision-instruct-model-card", "developerId": "meta", "title": "Llama-3.2-90B-Vision-Instruct Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Llama-3.2-90B-Vision-Instruct" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "meta-llama-3-3-70b-instruct-model-card", "sourceId": "official-meta-llama-3-3-70b-instruct-model-card", "developerId": "meta", "title": "Llama-3.3-70B-Instruct Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Llama-3.3-70B-Instruct" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "meta-llama-4-maverick-17b-128e-instruct-model-card", "sourceId": "official-meta-llama-4-maverick-17b-128e-instruct-model-card", "developerId": "meta", "title": "Llama-4-Maverick-17B-128E-Instruct Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Llama-4-Maverick-17B-128E-Instruct" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "meta-llama-4-scout-17b-16e-instruct-model-card", "sourceId": "official-meta-llama-4-scout-17b-16e-instruct-model-card", "developerId": "meta", "title": "Llama-4-Scout-17B-16E-Instruct Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Llama-4-Scout-17B-16E-Instruct" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "meta-meta-llama-3-70b-instruct-model-card", "sourceId": "official-meta-meta-llama-3-70b-instruct-model-card", "developerId": "meta", "title": "Meta-Llama-3-70B-Instruct Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Meta-Llama-3-70B-Instruct" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "meta-muse-glimmer-model-card", "sourceId": "official-meta-muse-glimmer-model-card", "developerId": "meta", "title": "Muse Glimmer Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Muse Glimmer" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Trust and Safety; Preparedness; Train-Time Mitigations", "locator": "Original source → Trust and Safety; Preparedness; Train-Time Mitigations", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Developer says this smaller model falls outside its frontier-model definition, while still assessing preparedness and local-agent risks. Do not present it as a frontier flagship.", "zh": "开发者称该小模型不属于其前沿模型定义，但仍评估准备度及本地代理风险；不应呈现为前沿旗舰。" } } ], "dateEvidence": [] }, { "id": "meta-muse-spark-1-2-evaluation-report", "sourceId": "official-meta-muse-spark-1-2-evaluation-report", "developerId": "meta", "title": "Muse Spark 1.2 Multimodal Evaluation Methodology", "documentType": "evaluation-methodology", "publicationDate": null, "updatedDate": null, "models": [ "Muse Spark 1.2" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Overall Methodology", "locator": "Original source → Overall Methodology", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Retrieved document is multimodal capability methodology, with anti-cheating restrictions; it is not a comprehensive frontier-risk report.", "zh": "取得文档为多模态能力评估方法，含防作弊限制，并非完整前沿风险报告。" } } ], "dateEvidence": [] }, { "id": "meta-muse-spark-1-3-evaluation-report", "sourceId": "official-meta-muse-spark-1-3-evaluation-report", "developerId": "meta", "title": "Muse Spark 1.3 Evaluation Methodology", "documentType": "evaluation-methodology", "publicationDate": null, "updatedDate": null, "models": [ "Muse Spark 1.3" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Evaluation Methodology", "locator": "Original source → Evaluation Methodology", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Retrieved document primarily describes capability-evaluation methodology; keep separate from full safety and preparedness reports.", "zh": "取得文档主要描述能力评估方法，须与完整安全与准备度报告区分。" } } ], "dateEvidence": [] }, { "id": "meta-muse-spark-contemplating-safety-preparedness-report", "sourceId": "official-meta-muse-spark-contemplating-safety-preparedness-report", "developerId": "meta", "title": "Muse Spark Contemplating Safety & Preparedness Report", "documentType": "safety-preparedness-report", "publicationDate": null, "updatedDate": null, "models": [ "Muse Spark Contemplating" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Preparedness and behavior assessment", "locator": "Original source → Preparedness and behavior assessment", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Separate deployment configuration/report; conclusions should not be pooled with baseline Muse Spark or later API-enabled releases.", "zh": "独立部署配置及报告；结论不能与基础Muse Spark或后续开放API版本合并。" } } ], "dateEvidence": [] }, { "id": "meta-muse-spark-open-evaluation-methodology", "sourceId": "official-meta-muse-spark-open-evaluation-methodology", "developerId": "meta", "title": "Muse Spark Open Evaluation Methodology", "documentType": "evaluation-methodology", "publicationDate": null, "updatedDate": null, "models": [ "Muse Spark Open" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Overall Methodology; safety benchmark descriptions", "locator": "Original source → Overall Methodology; safety benchmark descriptions", "protocolNames": [ "FORTRESS" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Methodology distinguishes benchmark-provider results, developer reruns and launch-post self-reports; comparisons are not uniformly matched experiments.", "zh": "方法说明区分基准机构结果、开发者复测与发布自报，比较并非统一匹配实验。" } } ], "dateEvidence": [] }, { "id": "meta-muse-spark-open-model-card", "sourceId": "official-meta-muse-spark-open-model-card", "developerId": "meta", "title": "Muse Spark Open Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Muse Spark Open" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Safety and Preparedness; Considerations and Limitations", "locator": "Original source → Safety and Preparedness; Considerations and Limitations", "protocolNames": [ "FORTRESS", "StrongREJECT v2" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Open-weight card covers content safety, agentic actions, privacy and preparedness. Its additional post-training does not recreate hosted system guardrails; fuller evaluation report was forthcoming.", "zh": "开放权重卡涵盖内容、代理行为、隐私与准备度；额外后训练不能替代托管系统防护，更完整报告仍待发布。" } } ], "dateEvidence": [] }, { "id": "meta-muse-spark-safety-preparedness-report-meta-original", "sourceId": "official-meta-muse-spark-safety-preparedness-report-meta-original", "developerId": "meta", "title": "Muse Spark Safety & Preparedness Report (Meta-hosted May 26 revision)", "documentType": "safety-preparedness-report", "publicationDate": null, "updatedDate": "2026-05-26", "models": [ "Muse Spark" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Change Log and 4.1.1 Instruction Hierarchy", "locator": "PDF front-matter change log and §4.1.1", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "This Meta-hosted revision corrects an instruction-hierarchy implementation bug; the report says the assessment was unchanged. It differs from the later arXiv HTML revision.", "zh": "该Meta托管修订纠正指令层级测试实现问题，报告称评估未变；与后续arXiv HTML修订不同。" } } ], "dateEvidence": [ { "field": "updatedDate", "value": "2026-05-26", "source": "PDF Date and Change Log; original first-publication date not established." } ] }, { "id": "meta-muse-wonder-model-card", "sourceId": "official-meta-muse-wonder-model-card", "developerId": "meta", "title": "Muse Wonder Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": "2026-10-23", "models": [ "Muse Wonder", "Muse Wonder Full" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "4 Risk assessments under the Meta Superintelligence Scaling Framework", "locator": "PDF pp.7–10", "protocolNames": [ "CyberGym", "ExploitBench", "ExploitGym", "MASK", "Agentic Misalignment" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Most capable Full variant bounds family assessment. Developer does not rule out high unmitigated biochemical capability; its cyber and loss-of-control judgment is moderate or lower under its own framework.", "zh": "以最强Full变体评估整个系列；开发者不排除缓解前高生化能力，并在自身框架下评为网络及失控风险中等或更低。" } }, { "heading": "Updated ExploitGym Evaluations", "locator": "PDF p.7, note dated October 23, 2026", "protocolNames": [ "ExploitGym" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Corrected harness and intended-flag reporting changed the published result. Use this revision, not the earlier value, for any future comparison.", "zh": "修正测试框架与预期标志计数后公布结果发生变化；未来比较应使用本修订而非旧值。" } }, { "heading": "5 Trust and Safety; 6 Considerations and Limitations", "locator": "PDF pp.11–12", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Card states that tests cannot cover all situations and recommends context-specific evaluation and additional system guardrails. Full safety report remained described as forthcoming in release copy.", "zh": "卡片说明测试无法覆盖所有场景，建议按用例评估并增加系统防护；发布文案仍称完整安全报告即将发布。" } } ], "dateEvidence": [ { "field": "developerReleaseDate", "value": "2026-10-12", "source": "PDF p.1, Date of Release" }, { "field": "updatedDate", "value": "2026-10-23", "source": "PDF p.7, Updated ExploitGym Evaluations note" } ] }, { "id": "meta-the-llama-3-herd-of-models", "sourceId": "official-meta-the-llama-3-herd-of-models", "developerId": "meta", "title": "The Llama 3 Herd of Models", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "Llama 3", "Llama 3.1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2407.21783 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "meta-muse-spark-safety-preparedness-report", "sourceId": "official-meta-muse-spark-safety-preparedness-report", "developerId": "meta", "title": "Muse Spark Safety & Preparedness Report", "documentType": "safety-preparedness-report", "publicationDate": "2026-05-14", "updatedDate": "2026-08-24", "models": [ "Muse Spark" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "2.1 Chemical & Biological Risks", "locator": "https://arxiv.org/html/2606.12429#S2.SS1", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Developer separates high pre-mitigation capability from residual risk of the Meta AI deployment; no numerical biology results or task content extracted.", "zh": "开发者区分缓解前高能力与Meta AI部署的剩余风险；未提取生物数值或任务内容。" } }, { "heading": "2.2 Cybersecurity", "locator": "https://arxiv.org/html/2606.12429#S2.SS2", "protocolNames": [ "Cybench", "CyberGym", "CyScenarioBench", "CyberSecEval" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Covers knowledge, agentic capability, refusals and insecure-code propensity; public and private tasks have different limits.", "zh": "涵盖知识、代理能力、拒绝与不安全代码倾向；公开和私有任务的推论限制不同。" } }, { "heading": "2.3 Loss of Control", "locator": "https://arxiv.org/html/2606.12429#S2.SS3", "protocolNames": [ "MASK", "Agentic Misalignment" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Assessment integrates evaluation reliability, monitorability and propensities; future-facing agentic tests exceed the initial Meta AI deployment context.", "zh": "综合评测可靠性、可监测性与倾向；前瞻代理测试超出最初Meta AI部署场景。" } }, { "heading": "3.2 Agent Robustness", "locator": "https://arxiv.org/html/2606.12429#S3.SS2", "protocolNames": [ "SIREN", "AgentDojo" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Indirect prompt injection tests use benign users and realistic scaffolds; residual attack susceptibility remains.", "zh": "间接提示注入测试使用善意用户与真实化框架；仍存在攻击脆弱性。" } }, { "heading": "4.1.5 Honesty", "locator": "https://arxiv.org/html/2606.12429#S4.SS1.SSS5", "protocolNames": [ "MASK", "DeceptionBench" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Distinguishes known-belief dishonesty from hallucination and third-party deceptive assistance.", "zh": "区分明知事实的失实陈述、幻觉和协助欺骗第三方。" } }, { "heading": "4.1.10 Scheming and 4.1.11 Evaluation Awareness", "locator": "https://arxiv.org/html/2606.12429#S4.SS1.SSS11", "protocolNames": [ "Alignment Faking", "Agentic Misalignment" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Apollo observed unusually high evaluation awareness. Follow-up behavioral impact was limited to a subset and did not inform hazardous-capability launch decisions; that does not prove no awareness effect.", "zh": "Apollo观察到高评测意识；复核发现行为影响限于部分测试且未涉及危险能力发布决策，但不能据此证明没有意识效应。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-05-14", "source": "https://arxiv.org/abs/2606.12429 → Submitted on / v1 history" }, { "field": "updatedDate", "value": "2026-08-24", "source": "HTML date and Change Log" } ] }, { "id": "meta-introducing-muse-spark-1-1", "sourceId": "official-meta-introducing-muse-spark-1-1", "developerId": "meta", "title": "Introducing Muse Spark 1.1", "documentType": "release-safety-summary", "publicationDate": "2026-07-09", "updatedDate": null, "models": [ "Muse Spark 1.1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-07-09", "source": "Official release blog masthead; applies to this blog, not the linked report." } ] }, { "id": "meta-muse-spark-1-1-evaluation-report", "sourceId": "official-meta-muse-spark-1-1-evaluation-report", "developerId": "meta", "title": "Muse Spark 1.1 Evaluation Report", "documentType": "evaluation-report", "publicationDate": "2026-07-09", "updatedDate": null, "models": [ "Muse Spark 1.1" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "2 Preparedness Assessment", "locator": "PDF §§2.1–2.3", "protocolNames": [ "Cybench", "CyberGym", "ExploitGym", "CyScenarioBench", "SHADE-Arena", "MASK" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "API agentic affordances expand the assessment context. Developer cannot rule out high pre-mitigation cyber and biochemical capability; residual-risk judgment includes safeguards.", "zh": "API代理能力扩大评估范围；开发者无法排除缓解前高网络与生化能力，剩余风险判断包含防护。" } }, { "heading": "3 Adversarial Robustness", "locator": "PDF §3", "protocolNames": [ "StrongREJECT v2", "FORTRESS", "AgentHarm Verified", "AgentDojo", "SWE-PI", "GraySwan ART" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Direct misuse, prompt injection and false refusals are evaluated separately.", "zh": "分别评估直接滥用、提示注入与错误拒绝。" } }, { "heading": "4 Model Behavior", "locator": "PDF §4", "protocolNames": [ "SAVE-Bench", "MASK", "DeceptionBench", "Agentic Misalignment" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Honesty, risk escalation, evaluation awareness and automated alignment audits are distinct behavior constructs; artificial scenario results have limited deployment generality.", "zh": "诚实性、风险升级、评测意识与自动对齐审计属于不同构念；模拟场景向部署泛化有限。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-07-09", "source": "PDF Date on opening page; official launch blog also dated July 9." } ] }, { "id": "meta-the-multimodal-intelligence-of-muse-spark-1-2", "sourceId": "official-meta-the-multimodal-intelligence-of-muse-spark-1-2", "developerId": "meta", "title": "The Multimodal Intelligence of Muse Spark 1.2", "documentType": "release-safety-summary", "publicationDate": "2026-08-20", "updatedDate": null, "models": [ "Muse Spark 1.2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-08-20", "source": "Official release blog masthead; applies to this blog, not the linked report." } ] }, { "id": "meta-introducing-muse-spark-1-3", "sourceId": "official-meta-introducing-muse-spark-1-3", "developerId": "meta", "title": "Introducing Muse Spark 1.3", "documentType": "release-safety-summary", "publicationDate": "2026-09-02", "updatedDate": null, "models": [ "Muse Spark 1.3" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-09-02", "source": "Official release blog masthead; applies to this blog, not the linked report." } ] }, { "id": "meta-muse-wonder-a-frontier-model-for-personal-professional-ai-agents", "sourceId": "official-meta-muse-wonder-a-frontier-model-for-personal-professional-ai-agents", "developerId": "meta", "title": "Muse Wonder: A Frontier Model For Personal & Professional AI Agents", "documentType": "release-safety-summary", "publicationDate": "2026-10-12", "updatedDate": null, "models": [ "Muse Wonder", "Muse Wonder Full" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-10-12", "source": "Official release blog masthead; applies to this blog, not the linked report." } ] }, { "id": "meta-introducing-muse-spark-open", "sourceId": "official-meta-introducing-muse-spark-open", "developerId": "meta", "title": "Introducing Muse Spark Open", "documentType": "release-safety-summary", "publicationDate": "2026-10-15", "updatedDate": null, "models": [ "Muse Spark Open" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-10-15", "source": "Official release blog masthead; applies to this blog, not the linked report." } ] }, { "id": "mistral-codestral-official-technical-documentation", "sourceId": "official-mistral-codestral-official-technical-documentation", "developerId": "mistral", "title": "Codestral Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Codestral" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-devstral-2-official-technical-documentation", "sourceId": "official-mistral-devstral-2-official-technical-documentation", "developerId": "mistral", "title": "Devstral 2 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Devstral 2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-devstral-medium-1-official-technical-documentation", "sourceId": "official-mistral-devstral-medium-1-official-technical-documentation", "developerId": "mistral", "title": "Devstral Medium 1 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Devstral Medium 1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-devstral-small-1-official-technical-documentation", "sourceId": "official-mistral-devstral-small-1-official-technical-documentation", "developerId": "mistral", "title": "Devstral Small 1 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Devstral Small 1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-devstral-small-2-official-technical-documentation", "sourceId": "official-mistral-devstral-small-2-official-technical-documentation", "developerId": "mistral", "title": "Devstral Small 2 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Devstral Small 2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-leanstral-official-technical-documentation", "sourceId": "official-mistral-leanstral-official-technical-documentation", "developerId": "mistral", "title": "Leanstral Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Leanstral" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-magistral-medium-1-official-technical-documentation", "sourceId": "official-mistral-magistral-medium-1-official-technical-documentation", "developerId": "mistral", "title": "Magistral Medium 1 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Magistral Medium 1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-magistral-small-1-official-technical-documentation", "sourceId": "official-mistral-magistral-small-1-official-technical-documentation", "developerId": "mistral", "title": "Magistral Small 1 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Magistral Small 1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-magistral-small-2506-model-card", "sourceId": "official-mistral-magistral-small-2506-model-card", "developerId": "mistral", "title": "Magistral-Small-2506 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Magistral-Small-2506" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mathstral-7b-official-technical-documentation", "sourceId": "official-mistral-mathstral-7b-official-technical-documentation", "developerId": "mistral", "title": "Mathstral 7B Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mathstral 7B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-ministral-1-3b-official-technical-documentation", "sourceId": "official-mistral-ministral-1-3b-official-technical-documentation", "developerId": "mistral", "title": "Ministral 1 - 3B Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Ministral 1 - 3B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-ministral-1-8b-official-technical-documentation", "sourceId": "official-mistral-ministral-1-8b-official-technical-documentation", "developerId": "mistral", "title": "Ministral 1 - 8B Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Ministral 1 - 8B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-ministral-3-14b-official-technical-documentation", "sourceId": "official-mistral-ministral-3-14b-official-technical-documentation", "developerId": "mistral", "title": "Ministral 3 - 14B Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Ministral 3 - 14B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-ministral-3-3b-official-technical-documentation", "sourceId": "official-mistral-ministral-3-3b-official-technical-documentation", "developerId": "mistral", "title": "Ministral 3 - 3B Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Ministral 3 - 3B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-ministral-3-8b-official-technical-documentation", "sourceId": "official-mistral-ministral-3-8b-official-technical-documentation", "developerId": "mistral", "title": "Ministral 3 - 8B Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Ministral 3 - 8B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-7b", "sourceId": "official-mistral-mistral-7b", "developerId": "mistral", "title": "Mistral 7B", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "Mistral 7B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2310.06825 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "mistral-mistral-7b-official-technical-documentation", "sourceId": "official-mistral-mistral-7b-official-technical-documentation", "developerId": "mistral", "title": "Mistral 7B Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral 7B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-embed-official-technical-documentation", "sourceId": "official-mistral-mistral-embed-official-technical-documentation", "developerId": "mistral", "title": "Mistral Embed Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Embed" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-large-1-0-official-technical-documentation", "sourceId": "official-mistral-mistral-large-1-0-official-technical-documentation", "developerId": "mistral", "title": "Mistral Large 1.0 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Large 1.0" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-large-2-0-official-technical-documentation", "sourceId": "official-mistral-mistral-large-2-0-official-technical-documentation", "developerId": "mistral", "title": "Mistral Large 2.0 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Large 2.0" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-large-3-official-technical-documentation", "sourceId": "official-mistral-mistral-large-3-official-technical-documentation", "developerId": "mistral", "title": "Mistral Large 3 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Large 3" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-large-4-official-technical-documentation", "sourceId": "official-mistral-mistral-large-4-official-technical-documentation", "developerId": "mistral", "title": "Mistral Large 4 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Large 4" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Model Lifecycle", "locator": "Original source → Model Lifecycle", "protocolNames": [], "disposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established", "notes": { "en": "Official table verifies release on 2026-10-06. The fetched page has no retrievable technical/safety-report content; HF card access returned 401.", "zh": "官方表确认2026-10-06发布；抓取页面没有可读技术/安全报告内容，HF卡访问返回401。" } } ], "dateEvidence": [] }, { "id": "mistral-mistral-medium-1-official-technical-documentation", "sourceId": "official-mistral-mistral-medium-1-official-technical-documentation", "developerId": "mistral", "title": "Mistral Medium 1 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Medium 1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-medium-3-0-official-technical-documentation", "sourceId": "official-mistral-mistral-medium-3-0-official-technical-documentation", "developerId": "mistral", "title": "Mistral Medium 3.0 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Medium 3.0" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-moderation-official-technical-documentation", "sourceId": "official-mistral-mistral-moderation-official-technical-documentation", "developerId": "mistral", "title": "Mistral Moderation Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Moderation" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-nemo-12b-official-technical-documentation", "sourceId": "official-mistral-mistral-nemo-12b-official-technical-documentation", "developerId": "mistral", "title": "Mistral Nemo 12B Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Nemo 12B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-next-official-technical-documentation", "sourceId": "official-mistral-mistral-next-official-technical-documentation", "developerId": "mistral", "title": "Mistral Next Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Next" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-ocr-3-official-technical-documentation", "sourceId": "official-mistral-mistral-ocr-3-official-technical-documentation", "developerId": "mistral", "title": "Mistral OCR 3 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral OCR 3" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-ocr-4-official-technical-documentation", "sourceId": "official-mistral-mistral-ocr-4-official-technical-documentation", "developerId": "mistral", "title": "Mistral OCR 4 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral OCR 4" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-ocr-official-technical-documentation", "sourceId": "official-mistral-mistral-ocr-official-technical-documentation", "developerId": "mistral", "title": "Mistral OCR Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral OCR" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-saba-official-technical-documentation", "sourceId": "official-mistral-mistral-saba-official-technical-documentation", "developerId": "mistral", "title": "Mistral Saba Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Saba" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-small-1-0-official-technical-documentation", "sourceId": "official-mistral-mistral-small-1-0-official-technical-documentation", "developerId": "mistral", "title": "Mistral Small 1.0 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Small 1.0" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-small-2-0-official-technical-documentation", "sourceId": "official-mistral-mistral-small-2-0-official-technical-documentation", "developerId": "mistral", "title": "Mistral Small 2.0 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Small 2.0" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-small-3-0-official-technical-documentation", "sourceId": "official-mistral-mistral-small-3-0-official-technical-documentation", "developerId": "mistral", "title": "Mistral Small 3.0 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Small 3.0" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-small-4-official-technical-documentation", "sourceId": "official-mistral-mistral-small-4-official-technical-documentation", "developerId": "mistral", "title": "Mistral Small 4 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mistral Small 4" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-7b-instruct-v0-1-model-card", "sourceId": "official-mistral-mistral-7b-instruct-v0-1-model-card", "developerId": "mistral", "title": "Mistral-7B-Instruct-v0.1 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Mistral-7B-Instruct-v0.1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-large-3-675b-instruct-2512-model-card", "sourceId": "official-mistral-mistral-large-3-675b-instruct-2512-model-card", "developerId": "mistral", "title": "Mistral-Large-3-675B-Instruct-2512 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Mistral-Large-3-675B-Instruct-2512" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-large-instruct-2407-model-card", "sourceId": "official-mistral-mistral-large-instruct-2407-model-card", "developerId": "mistral", "title": "Mistral-Large-Instruct-2407 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Mistral-Large-Instruct-2407" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-large-instruct-2411-model-card", "sourceId": "official-mistral-mistral-large-instruct-2411-model-card", "developerId": "mistral", "title": "Mistral-Large-Instruct-2411 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Mistral-Large-Instruct-2411" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mistral-small-4-119b-2603-model-card", "sourceId": "official-mistral-mistral-small-4-119b-2603-model-card", "developerId": "mistral", "title": "Mistral-Small-4-119B-2603 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Mistral-Small-4-119B-2603" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mixtral-8b-x-7b-official-technical-documentation", "sourceId": "official-mistral-mixtral-8b-x-7b-official-technical-documentation", "developerId": "mistral", "title": "Mixtral 8B x 7B Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mixtral 8B x 7B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mixtral-8x22b-official-technical-documentation", "sourceId": "official-mistral-mixtral-8x22b-official-technical-documentation", "developerId": "mistral", "title": "Mixtral 8x22B Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Mixtral 8x22B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mixtral-8x22b-instruct-v0-1-model-card", "sourceId": "official-mistral-mixtral-8x22b-instruct-v0-1-model-card", "developerId": "mistral", "title": "Mixtral-8x22B-Instruct-v0.1 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Mixtral-8x22B-Instruct-v0.1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-mixtral-8x7b-instruct-v0-1-model-card", "sourceId": "official-mistral-mixtral-8x7b-instruct-v0-1-model-card", "developerId": "mistral", "title": "Mixtral-8x7B-Instruct-v0.1 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Mixtral-8x7B-Instruct-v0.1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-pixtral-12b-official-technical-documentation", "sourceId": "official-mistral-pixtral-12b-official-technical-documentation", "developerId": "mistral", "title": "Pixtral 12B Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Pixtral 12B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-pixtral-large-official-technical-documentation", "sourceId": "official-mistral-pixtral-large-official-technical-documentation", "developerId": "mistral", "title": "Pixtral Large Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Pixtral Large" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-robostral-navigate-official-technical-documentation", "sourceId": "official-mistral-robostral-navigate-official-technical-documentation", "developerId": "mistral", "title": "Robostral Navigate Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Robostral Navigate" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-shieldstral", "sourceId": "official-mistral-shieldstral", "developerId": "mistral", "title": "Shieldstral", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "Shieldstral" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Safety classification, refusal detection and multimodal safety evaluations", "locator": "Original source → Safety classification, refusal detection and multimodal safety evaluations", "protocolNames": [ "WildGuardTest", "HarmBench", "XSTest" ], "disposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established", "notes": { "en": "Policy-adaptive safety classifier; its moderation results do not establish the safety of Mistral Large 4.", "zh": "策略自适应安全分类器；其审核结果不能证明Mistral Large 4安全。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2607.25857 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "mistral-shieldstral-official-technical-documentation", "sourceId": "official-mistral-shieldstral-official-technical-documentation", "developerId": "mistral", "title": "Shieldstral Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Shieldstral" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-shieldstral-1-0-3b-model-card", "sourceId": "official-mistral-shieldstral-1-0-3b-model-card", "developerId": "mistral", "title": "Shieldstral-1.0-3B Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Shieldstral-1.0-3B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-timestral-official-technical-documentation", "sourceId": "official-mistral-timestral-official-technical-documentation", "developerId": "mistral", "title": "Timestral Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Timestral" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-voxtral-2-official-technical-documentation", "sourceId": "official-mistral-voxtral-2-official-technical-documentation", "developerId": "mistral", "title": "Voxtral 2 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Voxtral 2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-voxtral-3-official-technical-documentation", "sourceId": "official-mistral-voxtral-3-official-technical-documentation", "developerId": "mistral", "title": "Voxtral 3 Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Voxtral 3" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-voxtral-mini-tts-official-technical-documentation", "sourceId": "official-mistral-voxtral-mini-tts-official-technical-documentation", "developerId": "mistral", "title": "Voxtral Mini TTS Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Voxtral Mini TTS" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "mistral-voxtral-small-official-technical-documentation", "sourceId": "official-mistral-voxtral-small-official-technical-documentation", "developerId": "mistral", "title": "Voxtral Small Official Technical Documentation", "documentType": "model-lifecycle-page", "publicationDate": null, "updatedDate": null, "models": [ "Voxtral Small" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "moonshot-kimi-k2-technical-report", "sourceId": "official-moonshot-kimi-k2-technical-report", "developerId": "moonshot", "title": "Kimi K2 Technical Report", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "Kimi K2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "moonshot-kimi-k2-5-technical-report", "sourceId": "official-moonshot-kimi-k2-5-technical-report", "developerId": "moonshot", "title": "Kimi K2.5 Technical Report", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "Kimi K2.5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2602.02276 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "moonshot-kimi-k3-technical-report", "sourceId": "official-moonshot-kimi-k3-technical-report", "developerId": "moonshot", "title": "Kimi K3 Technical Report", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "Kimi K3" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "4.2 Agentic Reinforcement Learning; 5 training infrastructure", "locator": "Original source → 4.2 Agentic Reinforcement Learning; 5 training infrastructure", "protocolNames": [], "disposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established", "notes": { "en": "Training discusses hidden verifiers and sandbox isolation. The reviewed report contains no dedicated catastrophic-risk assessment; Neo findings remain independent sources.", "zh": "训练部分讨论隐藏验证器与沙箱隔离；所审报告没有独立的灾难风险评估章节，Neo结论仍属独立来源。" } } ], "dateEvidence": [] }, { "id": "moonshot-kimi-k1-5-scaling-reinforcement-learning-with-llms", "sourceId": "official-moonshot-kimi-k1-5-scaling-reinforcement-learning-with-llms", "developerId": "moonshot", "title": "Kimi k1.5: Scaling Reinforcement Learning with LLMs", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "Kimi k1.5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2501.12599 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "moonshot-kimi-k2-instruct-model-card", "sourceId": "official-moonshot-kimi-k2-instruct-model-card", "developerId": "moonshot", "title": "Kimi-K2-Instruct Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Kimi-K2-Instruct" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "moonshot-kimi-k2-instruct-0905-model-card", "sourceId": "official-moonshot-kimi-k2-instruct-0905-model-card", "developerId": "moonshot", "title": "Kimi-K2-Instruct-0905 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Kimi-K2-Instruct-0905" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "moonshot-kimi-k2-thinking-model-card", "sourceId": "official-moonshot-kimi-k2-thinking-model-card", "developerId": "moonshot", "title": "Kimi-K2-Thinking Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Kimi-K2-Thinking" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "moonshot-kimi-k2-5-model-card", "sourceId": "official-moonshot-kimi-k2-5-model-card", "developerId": "moonshot", "title": "Kimi-K2.5 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Kimi-K2.5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "moonshot-kimi-k3-model-card", "sourceId": "official-moonshot-kimi-k3-model-card", "developerId": "moonshot", "title": "Kimi-K3 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Kimi-K3" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "openai-dall-e-3-system-card-pdf-original", "sourceId": "official-openai-dall-e-3-system-card-pdf-original", "developerId": "openai", "title": "DALL-E 3 System Card", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "DALL-E 3" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "openai-dall-e-2-preview-risks-and-limitations", "sourceId": "official-openai-dall-e-2-preview-risks-and-limitations", "developerId": "openai", "title": "DALL·E 2 Preview - Risks and Limitations", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "DALL·E 2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "openai-gpt-4-system-card-pdf-original", "sourceId": "official-openai-gpt-4-system-card-pdf-original", "developerId": "openai", "title": "GPT-4 System Card", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "GPT-4" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "openai-gpt-4-5-system-card-february-original", "sourceId": "official-openai-gpt-4-5-system-card-february-original", "developerId": "openai", "title": "GPT-4.5 System Card February Original", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "GPT-4.5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "openai-gpt-4v-system-card-pdf-original", "sourceId": "official-openai-gpt-4v-system-card-pdf-original", "developerId": "openai", "title": "GPT-4V System Card PDF Original", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "GPT-4V" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "openai-gpt-4o-system-card-pdf-original", "sourceId": "official-openai-gpt-4o-system-card-pdf-original", "developerId": "openai", "title": "GPT-4o System Card", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "GPT-4o" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "openai-openai-o1-preview-system-card-pdf-original", "sourceId": "official-openai-openai-o1-preview-system-card-pdf-original", "developerId": "openai", "title": "OpenAI o1 Preview System Card PDF Original", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "o1-preview", "o1-mini" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "openai-openai-o1-system-card-pdf-original", "sourceId": "official-openai-openai-o1-system-card-pdf-original", "developerId": "openai", "title": "OpenAI o1 System Card", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "o1", "o1-mini" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "openai-openai-o3-and-o4-mini-system-card-pdf-original", "sourceId": "official-openai-openai-o3-and-o4-mini-system-card-pdf-original", "developerId": "openai", "title": "OpenAI o3 and o4-mini System Card", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "o3", "o4-mini" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "openai-openai-o3-mini-system-card-february-revision", "sourceId": "official-openai-openai-o3-mini-system-card-february-revision", "developerId": "openai", "title": "OpenAI o3-mini System Card February Revision", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "o3-mini" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "openai-operator-system-card-pdf-original", "sourceId": "official-openai-operator-system-card-pdf-original", "developerId": "openai", "title": "Operator System Card", "documentType": "system-card", "publicationDate": null, "updatedDate": null, "models": [ "Operator" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "openai-training-language-models-to-follow-instructions-with-human-feedback", "sourceId": "official-openai-training-language-models-to-follow-instructions-with-human-feedback", "developerId": "openai", "title": "Training language models to follow instructions with human feedback", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "InstructGPT" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2203.02155 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "openai-deep-research-system-card", "sourceId": "official-openai-deep-research-system-card", "developerId": "openai", "title": "Deep Research System Card", "documentType": "system-card", "publicationDate": "2025-02-25", "updatedDate": null, "models": [ "Deep Research" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-02-25", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-addendum-to-gpt-4o-system-card-native-image-generation", "sourceId": "official-openai-addendum-to-gpt-4o-system-card-native-image-generation", "developerId": "openai", "title": "Addendum to GPT-4o System Card: Native image generation", "documentType": "system-card-addendum", "publicationDate": "2025-03-25", "updatedDate": null, "models": [ "GPT-4" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-03-25", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-chatgpt-agent-system-card", "sourceId": "official-openai-chatgpt-agent-system-card", "developerId": "openai", "title": "ChatGPT Agent System Card", "documentType": "system-card", "publicationDate": "2025-07-17", "updatedDate": null, "models": [ "ChatGPT Agent" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-07-17", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-oss-120b-gpt-oss-20b-model-card", "sourceId": "official-openai-gpt-oss-120b-gpt-oss-20b-model-card", "developerId": "openai", "title": "gpt-oss-120b & gpt-oss-20b Model Card", "documentType": "system-card", "publicationDate": "2025-08-05", "updatedDate": null, "models": [ "gpt-oss-120b", "gpt-oss-20b" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-08-05", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-5-system-card", "sourceId": "official-openai-gpt-5-system-card", "developerId": "openai", "title": "GPT-5 System Card", "documentType": "system-card", "publicationDate": "2025-08-07", "updatedDate": null, "models": [ "GPT-5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-08-07", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-sora-2-system-card", "sourceId": "official-openai-sora-2-system-card", "developerId": "openai", "title": "Sora 2 System Card", "documentType": "system-card", "publicationDate": "2025-09-30", "updatedDate": null, "models": [ "Sora 2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-09-30", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-addendum-to-gpt-5-system-card-sensitive-conversations", "sourceId": "official-openai-addendum-to-gpt-5-system-card-sensitive-conversations", "developerId": "openai", "title": "Addendum to GPT-5 System Card: Sensitive Conversations", "documentType": "system-card-addendum", "publicationDate": "2025-10-27", "updatedDate": null, "models": [ "GPT-5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-10-27", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-5-1-instant-and-gpt-5-1-thinking-system-card-addendum", "sourceId": "official-openai-gpt-5-1-instant-and-gpt-5-1-thinking-system-card-addendum", "developerId": "openai", "title": "GPT-5.1 Instant and GPT-5.1 Thinking System Card Addendum", "documentType": "system-card-addendum", "publicationDate": "2025-11-12", "updatedDate": null, "models": [ "GPT-5.1 Instant", "GPT-5.1 Thinking" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-11-12", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-5-1-codex-max-system-card", "sourceId": "official-openai-gpt-5-1-codex-max-system-card", "developerId": "openai", "title": "GPT-5.1-Codex-Max System Card", "documentType": "system-card", "publicationDate": "2025-11-18", "updatedDate": null, "models": [ "GPT-5.1-Codex-Max" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-11-18", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-update-to-gpt-5-system-card-gpt-5-2", "sourceId": "official-openai-update-to-gpt-5-system-card-gpt-5-2", "developerId": "openai", "title": "Update to GPT-5 System Card: GPT-5.2", "documentType": "system-card-addendum", "publicationDate": "2025-12-11", "updatedDate": null, "models": [ "GPT-5", "GPT-5.2" ], "reviewLevel": "section-map-only", "sections": [ { "heading": "Preparedness, alignment and monitorability", "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections", "protocolNames": [ "Cyber range", "MLE-bench" ], "disposition": "section-map-reviewed; result-level-comparison-pending", "notes": { "en": "Comparator evaluation sections located; capability, refusal, monitoring and deployment safeguards are separate constructs. No numerical evidence is imported.", "zh": "已定位比较评估章节；能力、拒绝、监测及部署防护为不同构念，未导入数值证据。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2025-12-11", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-addendum-to-gpt-5-2-system-card-gpt-5-2-codex", "sourceId": "official-openai-addendum-to-gpt-5-2-system-card-gpt-5-2-codex", "developerId": "openai", "title": "Addendum to GPT-5.2 System Card: GPT-5.2-Codex", "documentType": "system-card-addendum", "publicationDate": "2025-12-18", "updatedDate": null, "models": [ "GPT-5.2", "GPT-5.2-Codex" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-12-18", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-5-3-codex-system-card", "sourceId": "official-openai-gpt-5-3-codex-system-card", "developerId": "openai", "title": "GPT-5.3-Codex System Card", "documentType": "system-card", "publicationDate": "2026-02-05", "updatedDate": null, "models": [ "GPT-5.3-Codex" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-02-05", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-5-3-instant-system-card", "sourceId": "official-openai-gpt-5-3-instant-system-card", "developerId": "openai", "title": "GPT-5.3 Instant System Card", "documentType": "system-card", "publicationDate": "2026-03-02", "updatedDate": null, "models": [ "GPT-5.3 Instant" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-03-02", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-5-4-thinking-system-card", "sourceId": "official-openai-gpt-5-4-thinking-system-card", "developerId": "openai", "title": "GPT-5.4 Thinking System Card", "documentType": "system-card", "publicationDate": "2026-03-05", "updatedDate": null, "models": [ "GPT-5.4 Thinking" ], "reviewLevel": "section-map-only", "sections": [ { "heading": "Preparedness, alignment and monitorability", "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections", "protocolNames": [ "Cyber range", "SWE-bench Verified", "MLE-bench" ], "disposition": "section-map-reviewed; result-level-comparison-pending", "notes": { "en": "Comparator evaluation sections located; capability, refusal, monitoring and deployment safeguards are separate constructs. No numerical evidence is imported.", "zh": "已定位比较评估章节；能力、拒绝、监测及部署防护为不同构念，未导入数值证据。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-03-05", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-chatgpt-images-2-0-system-card", "sourceId": "official-openai-chatgpt-images-2-0-system-card", "developerId": "openai", "title": "ChatGPT Images 2.0 System Card", "documentType": "system-card", "publicationDate": "2026-04-21", "updatedDate": null, "models": [], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-04-21", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-5-5-system-card", "sourceId": "official-openai-gpt-5-5-system-card", "developerId": "openai", "title": "GPT-5.5 System Card", "documentType": "system-card", "publicationDate": "2026-04-23", "updatedDate": null, "models": [ "GPT-5.5" ], "reviewLevel": "section-map-only", "sections": [ { "heading": "Preparedness, alignment and monitorability", "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections", "protocolNames": [ "Cyber range", "SWE-bench Verified", "MLE-bench" ], "disposition": "section-map-reviewed; result-level-comparison-pending", "notes": { "en": "Comparator evaluation sections located; capability, refusal, monitoring and deployment safeguards are separate constructs. No numerical evidence is imported.", "zh": "已定位比较评估章节；能力、拒绝、监测及部署防护为不同构念，未导入数值证据。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-04-23", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-5-5-instant-system-card", "sourceId": "official-openai-gpt-5-5-instant-system-card", "developerId": "openai", "title": "GPT-5.5 Instant System Card", "documentType": "system-card", "publicationDate": "2026-05-05", "updatedDate": null, "models": [ "GPT-5.5 Instant" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-05-05", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-rosalind-5-5-system-card", "sourceId": "official-openai-gpt-rosalind-5-5-system-card", "developerId": "openai", "title": "GPT-Rosalind-5.5 System Card", "documentType": "system-card", "publicationDate": "2026-06-03", "updatedDate": null, "models": [], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-06-03", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-5-6-preview-system-card", "sourceId": "official-openai-gpt-5-6-preview-system-card", "developerId": "openai", "title": "GPT-5.6 Preview System Card", "documentType": "system-card", "publicationDate": "2026-06-26", "updatedDate": null, "models": [ "GPT-5.6" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-06-26", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-live-system-card", "sourceId": "official-openai-gpt-live-system-card", "developerId": "openai", "title": "GPT-Live System Card", "documentType": "system-card", "publicationDate": "2026-07-08", "updatedDate": null, "models": [], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-07-08", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-5-6-system-card", "sourceId": "official-openai-gpt-5-6-system-card", "developerId": "openai", "title": "GPT-5.6 System Card", "documentType": "system-card", "publicationDate": "2026-07-09", "updatedDate": null, "models": [ "GPT-5.6" ], "reviewLevel": "section-map-only", "sections": [ { "heading": "Preparedness, alignment and monitorability", "locator": "Original HTML → named Preparedness / Alignment / CoT Monitorability sections", "protocolNames": [ "CyberGym", "ExploitBench", "Cyber range", "SWE-bench Verified", "MLE-bench" ], "disposition": "section-map-reviewed; result-level-comparison-pending", "notes": { "en": "Comparator evaluation sections located; capability, refusal, monitoring and deployment safeguards are separate constructs. No numerical evidence is imported.", "zh": "已定位比较评估章节；能力、拒绝、监测及部署防护为不同构念，未导入数值证据。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-07-09", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-5-6-august-updates", "sourceId": "official-openai-gpt-5-6-august-updates", "developerId": "openai", "title": "GPT-5.6 — August Updates", "documentType": "system-card-addendum", "publicationDate": "2026-08-06", "updatedDate": null, "models": [ "GPT-5.6" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-08-06", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-6-astra-system-card", "sourceId": "official-openai-gpt-6-astra-system-card", "developerId": "openai", "title": "GPT-6 Astra System Card", "documentType": "system-card", "publicationDate": "2026-09-03", "updatedDate": "2026-09-09", "models": [ "GPT-6 Astra", "GPT-6 Sol (September)", "GPT-6 Luna (September)", "dots" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "8 Alignment", "locator": "https://deploymentsafety.openai.com/gpt-6-astra#section.8", "protocolNames": [ "ExploitGym", "deployment simulation" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Tests include restriction following, deception, realistic work environments and unintended agent communication. Evaluated behavior is not a deployment incident rate.", "zh": "测试涵盖遵守限制、欺骗、真实化工作环境及非预期代理通信；评测结果不等于部署事故率。" } }, { "heading": "8.7 Verbalized Metagaming and Oversight Gaming", "locator": "https://deploymentsafety.openai.com/gpt-6-astra#subsection.8.7", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Card update clarifies that verbalized metagaming is detected in reasoning; oversight-gaming judgments rely on interpretation, not causal identification.", "zh": "卡片更新说明其检测对象为推理中表述的元博弈；监督博弈判断依赖解释而非因果识别。" } }, { "heading": "9 Monitorability", "locator": "https://deploymentsafety.openai.com/gpt-6-astra#section.9", "protocolNames": [ "CoT controllability", "monitor evasion", "Sneaky Sandbox Escape" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Separates ordinary and adversarial monitorability. Prompted evasion tests do not establish spontaneous deployed evasion.", "zh": "区分常规与对抗可监测性；诱导规避测试不能证明部署中自发规避。" } }, { "heading": "10 Preparedness", "locator": "https://deploymentsafety.openai.com/gpt-6-astra#section.10", "protocolNames": [ "ExploitBench", "SRE-Bench", "Sandbox Bench", "KernelGen 1P", "NanoGPT", "PostTrainBench Lite", "MLE-Bench Revised" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Developer assigns Critical cyber and High biological/chemical capability, with safeguards and trusted-access cases; AI self-improvement remains below its High threshold.", "zh": "开发者认定网络能力为Critical、生化能力为High，并配套防护及可信访问；AI自我改进未达其High阈值。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-09-03", "source": "Official document listing; original document date, not release date." }, { "field": "updatedDate", "value": "2026-09-09", "source": "§8.7 explicitly dated clarification; latest explicit card amendment verified in reviewed material, not an exhaustive revision history." } ] }, { "id": "openai-chatgpt-images-2-5-system-card", "sourceId": "official-openai-chatgpt-images-2-5-system-card", "developerId": "openai", "title": "ChatGPT Images 2.5 System Card", "documentType": "system-card", "publicationDate": "2026-09-08", "updatedDate": null, "models": [], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-09-08", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-addendum-to-gpt-6-astra-system-card-gpt-6-1-sol", "sourceId": "official-openai-addendum-to-gpt-6-astra-system-card-gpt-6-1-sol", "developerId": "openai", "title": "Addendum to GPT-6 Astra System Card: GPT-6.1 Sol", "documentType": "system-card-addendum", "publicationDate": "2026-09-29", "updatedDate": null, "models": [ "GPT-6.1 Sol" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "1 Introduction", "locator": "https://deploymentsafety.openai.com/gpt-6-1-sol#section.1", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Addendum evaluates GPT-6.1 Sol and applies Astra safeguards. Prior-model comparison values can reflect later versions rather than launch snapshots.", "zh": "附录评估GPT-6.1 Sol并沿用Astra防护；比较模型数据可能来自后续版本而非发布时快照。" } }, { "heading": "7 Alignment and 8 Monitorability", "locator": "https://deploymentsafety.openai.com/gpt-6-1-sol#section.7", "protocolNames": [ "ExploitGym", "CoT controllability", "monitor evasion" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Restriction following, deception, realistic deployment simulations and prompted monitor evasion require separate interpretations.", "zh": "遵守限制、欺骗、真实化部署模拟和诱导规避监测须分别解读。" } }, { "heading": "9 Preparedness", "locator": "https://deploymentsafety.openai.com/gpt-6-1-sol#section.9", "protocolNames": [ "ExploitBench", "KernelGen 1P" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Developer treats this model as Critical cyber and High biological/chemical capability; safeguards summary points back to Astra.", "zh": "开发者将其网络能力按Critical、生化能力按High处理；防护摘要指向Astra。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-09-29", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-6-sol-and-gpt-6-luna-october-2026-update", "sourceId": "official-openai-gpt-6-sol-and-gpt-6-luna-october-2026-update", "developerId": "openai", "title": "GPT-6 Sol and GPT-6 Luna: October 2026 update", "documentType": "system-card-addendum", "publicationDate": "2026-10-07", "updatedDate": null, "models": [ "GPT-6 Sol (October)", "GPT-6 Luna (October)" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "1 Introduction", "locator": "https://deploymentsafety.openai.com/gpt-6-october#section.1", "protocolNames": [], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "October ChatGPT versions differ from September versions still used in Codex and Work. Preserve the month-specific model identity.", "zh": "10月ChatGPT版本与Codex、Work继续使用的9月版本不同；保留月份级模型身份。" } }, { "heading": "4 Robustness and 7 Alignment", "locator": "https://deploymentsafety.openai.com/gpt-6-october#section.4", "protocolNames": [ "GPT-RED", "Blocker Deception", "Broken Search Tool" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Reports adaptive jailbreak, prompt-injection and restriction-following tests; severity review accompanies regressions.", "zh": "报告自适应越狱、提示注入及遵守限制测试，并对退化结果开展严重性复核。" } }, { "heading": "8 Preparedness", "locator": "https://deploymentsafety.openai.com/gpt-6-october#section.8", "protocolNames": [ "ExploitBench", "SEC-Bench Pro" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "October Sol/Luna are treated as High in cyber and biological/chemical domains; capability assessments use maximum reasoning, unlike baseline safety tests.", "zh": "10月Sol/Luna的网络与生化能力均按High处理；能力评估使用最高推理配置，与基础安全测试不同。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-10-07", "source": "Official document listing; original document date, not release date." } ] }, { "id": "openai-gpt-live-1-1-system-card", "sourceId": "official-openai-gpt-live-1-1-system-card", "developerId": "openai", "title": "GPT-Live-1.1 system card", "documentType": "system-card", "publicationDate": "2026-10-15", "updatedDate": null, "models": [], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-10-15", "source": "Official document listing; original document date, not release date." } ] }, { "id": "qwen-qwen-technical-report", "sourceId": "official-qwen-qwen-technical-report", "developerId": "qwen", "title": "Qwen Technical Report", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "Qwen" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2309.16609 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "qwen-qwen-72b-chat-model-card", "sourceId": "official-qwen-qwen-72b-chat-model-card", "developerId": "qwen", "title": "Qwen-72B-Chat Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Qwen-72B-Chat" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "qwen-qwen2-technical-report", "sourceId": "official-qwen-qwen2-technical-report", "developerId": "qwen", "title": "Qwen2 Technical Report", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "Qwen2" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2407.10671 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "qwen-qwen2-72b-instruct-model-card", "sourceId": "official-qwen-qwen2-72b-instruct-model-card", "developerId": "qwen", "title": "Qwen2-72B-Instruct Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Qwen2-72B-Instruct" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "qwen-qwen2-5-72b-instruct-model-card", "sourceId": "official-qwen-qwen2-5-72b-instruct-model-card", "developerId": "qwen", "title": "Qwen2.5-72B-Instruct Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Qwen2.5-72B-Instruct" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "qwen-qwen3-235b-a22b-model-card", "sourceId": "official-qwen-qwen3-235b-a22b-model-card", "developerId": "qwen", "title": "Qwen3-235B-A22B Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Qwen3-235B-A22B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "qwen-qwen3-5-397b-a17b-model-card", "sourceId": "official-qwen-qwen3-5-397b-a17b-model-card", "developerId": "qwen", "title": "Qwen3.5-397B-A17B Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Qwen3.5-397B-A17B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "qwen-qwen3-8-2-4t-a95b-model-card", "sourceId": "official-qwen-qwen3-8-2-4t-a95b-model-card", "developerId": "qwen", "title": "Qwen3.8-2.4T-A95B Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Qwen3.8-2.4T-A95B" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Benchmark Results; evaluation footnotes", "locator": "Original source → Benchmark Results; evaluation footnotes", "protocolNames": [ "NL2Repo-Bench", "PaperBench" ], "disposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established", "notes": { "en": "Footnotes document anti-reward-hacking controls and harness differences; no dedicated frontier-risk evaluation found.", "zh": "脚注说明防奖励投机控制与框架差异；未发现独立前沿风险评估。" } } ], "dateEvidence": [] }, { "id": "qwen-qwen3-8-flash-next-model-card", "sourceId": "official-qwen-qwen3-8-flash-next-model-card", "developerId": "qwen", "title": "Qwen3.8-Flash-Next Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Qwen3.8-Flash-Next" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "qwen-qwen3guard-technical-report", "sourceId": "official-qwen-qwen3guard-technical-report", "developerId": "qwen", "title": "Qwen3Guard Technical Report", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "Qwen3Guard" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "2 Safety Policy; benchmark evaluations", "locator": "Original source → 2 Safety Policy; benchmark evaluations", "protocolNames": [ "Qwen3GuardTest", "WildGuardTest" ], "disposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established", "notes": { "en": "A guardrail-classifier evaluation is governance/safeguard evidence, not an assessment of frontier base-model dangerous capability.", "zh": "防护分类器评估属于治理/防护证据，不能视为前沿基础模型危险能力评估。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2510.14276 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "qwen-qwen3guard-gen-8b-model-card", "sourceId": "official-qwen-qwen3guard-gen-8b-model-card", "developerId": "qwen", "title": "Qwen3Guard-Gen-8B Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Qwen3Guard-Gen-8B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "qwen-qwen3guard-stream-8b-model-card", "sourceId": "official-qwen-qwen3guard-stream-8b-model-card", "developerId": "qwen", "title": "Qwen3Guard-Stream-8B Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "Qwen3Guard-Stream-8B" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "qwen-qwen3-technical-report", "sourceId": "official-qwen-qwen3-technical-report", "developerId": "qwen", "title": "Qwen3 Technical Report", "documentType": "technical-report", "publicationDate": "2025-05-14", "updatedDate": null, "models": [ "Qwen3" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-05-14", "source": "https://arxiv.org/abs/2505.09388 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "qwen-qwen3-8-flash-next-technical-report", "sourceId": "official-qwen-qwen3-8-flash-next-technical-report", "developerId": "qwen", "title": "On the Design of Qwen3.8-Next Architecture: Evaluation, Efficiency, and Training Stability", "documentType": "technical-report", "publicationDate": "2026-08-26", "updatedDate": null, "models": [ "Qwen3.8-Flash-Next" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Architecture, evaluation and training-stability sections", "locator": "Original source → Architecture, evaluation and training-stability sections", "protocolNames": [], "disposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established", "notes": { "en": "Technical architecture and optimization report; no dedicated frontier-risk assessment in the reviewed source.", "zh": "技术架构与优化报告；所审来源没有独立前沿风险评估。" } } ], "dateEvidence": [] }, { "id": "xai-grok-4-model-card", "sourceId": "official-xai-grok-4-model-card", "developerId": "xai", "title": "Grok 4 Model Card", "documentType": "model-card", "publicationDate": "2025-08-20", "updatedDate": null, "models": [ "Grok 4" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-08-20", "source": "PDF title page" } ] }, { "id": "xai-grok-code-fast-1-model-card", "sourceId": "official-xai-grok-code-fast-1-model-card", "developerId": "xai", "title": "Grok Code Fast 1 Model Card", "documentType": "model-card", "publicationDate": "2025-08-26", "updatedDate": null, "models": [ "Grok Code Fast 1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-08-26", "source": "PDF title page" } ] }, { "id": "xai-grok-4-fast-model-card", "sourceId": "official-xai-grok-4-fast-model-card", "developerId": "xai", "title": "Grok 4 Fast Model Card", "documentType": "model-card", "publicationDate": "2025-09-19", "updatedDate": null, "models": [ "Grok 4 Fast" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2025-09-19", "source": "PDF title page" } ] }, { "id": "xai-grok-4-5-model-card", "sourceId": "official-xai-grok-4-5-model-card", "developerId": "xai", "title": "Grok 4.5 Model Card", "documentType": "model-card", "publicationDate": "2026-07-14", "updatedDate": "2026-07-20", "models": [ "Grok 4.5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-07-14", "source": "PDF title page" } ] }, { "id": "xai-grok-4-6-model-card", "sourceId": "official-xai-grok-4-6-model-card", "developerId": "xai", "title": "Grok 4.6 Model Card", "documentType": "model-card", "publicationDate": "2026-08-12", "updatedDate": "2026-08-17", "models": [ "Grok 4.6" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": "2026-08-12", "source": "PDF title page" } ] }, { "id": "xai-grok-4-7-model-card", "sourceId": "official-xai-grok-4-7-model-card", "developerId": "xai", "title": "Grok 4.7 Model Card", "documentType": "model-card", "publicationDate": "2026-09-21", "updatedDate": "2026-09-21", "models": [ "Grok 4.7" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "6 Cyber capabilities and safeguards", "locator": "PDF pp.17–19 (printed 16–18)", "protocolNames": [ "CyberGym", "CVE-Bench", "HackerBench v0.3", "CathedralBench" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Capability probes use unrestricted settings; safeguards/refusals are evaluated separately. Peer-card values are imported comparisons, not necessarily matched reruns.", "zh": "能力探测使用无限制配置，防护/拒绝另测；其他模型卡数据不一定为匹配条件复测。" } }, { "heading": "7 Biological and chemical capabilities and safeguards", "locator": "PDF pp.20–23 (printed 19–22)", "protocolNames": [ "BioSecBench", "VCT", "Biosecurity VCT", "BioUseBench", "WMDP", "LAB-Bench", "ProtocolQA", "BixBench" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Knowledge/capability and refusal are explicitly different signals; only protocol metadata and bounded high-level conclusions retained.", "zh": "知识/能力与拒绝是不同信号；仅保留协议元数据与有边界的高层结论。" } }, { "heading": "8 Jailbreaks and robustness", "locator": "PDF p.23 (printed 22)", "protocolNames": [ "StrongREJECT", "Crescendo" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "Production safeguards are stress-tested under single-turn, multiturn and longer attacks; residual failures motivate monitoring.", "zh": "生产防护接受单轮、多轮与长程攻击测试；剩余失败仍需监测。" } }, { "heading": "11 Behaviors", "locator": "PDF pp.26–27 (printed 25–26)", "protocolNames": [ "MASK-Rectified" ], "disposition": "source-section-reviewed; numerical-comparison-not-assessed", "notes": { "en": "MASK-Rectified changes grading relative to MASK, so its honesty result cannot be pooled with ordinary MASK.", "zh": "MASK-Rectified改变MASK评分方式，诚实性结果不能与普通MASK合并。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": "2026-09-21", "source": "PDF title page" } ] }, { "id": "zai-glm-4-5-model-card", "sourceId": "official-zai-glm-4-5-model-card", "developerId": "zai", "title": "GLM-4.5 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "GLM-4.5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "zai-glm-4-5-agentic-reasoning-and-coding-arc-foundation-models", "sourceId": "official-zai-glm-4-5-agentic-reasoning-and-coding-arc-foundation-models", "developerId": "zai", "title": "GLM-4.5: Agentic, Reasoning, and Coding (ARC) Foundation Models", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "GLM-4.5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2508.06471 → Submitted on; this is arXiv submission, not a model release." } ] }, { "id": "zai-glm-4-6-model-card", "sourceId": "official-zai-glm-4-6-model-card", "developerId": "zai", "title": "GLM-4.6 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "GLM-4.6" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "zai-glm-4-7-model-card", "sourceId": "official-zai-glm-4-7-model-card", "developerId": "zai", "title": "GLM-4.7 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "GLM-4.7" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "zai-glm-5-model-card", "sourceId": "official-zai-glm-5-model-card", "developerId": "zai", "title": "GLM-5 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "GLM-5" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "zai-glm-5-1-model-card", "sourceId": "official-zai-glm-5-1-model-card", "developerId": "zai", "title": "GLM-5.1 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "GLM-5.1" ], "reviewLevel": "source-indexed", "sections": [], "dateEvidence": [] }, { "id": "zai-glm-5-2-model-card", "sourceId": "official-zai-glm-5-2-model-card", "developerId": "zai", "title": "GLM-5.2 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "GLM-5.2" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Benchmark; Footnote", "locator": "Original source → Benchmark; Footnote", "protocolNames": [ "SWE-bench Pro", "DeepSWE", "MCP-Atlas" ], "disposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established", "notes": { "en": "General capability comparisons and harness notes; no dedicated frontier-risk evaluation or safeguard-effectiveness section located in this card.", "zh": "一般能力比较与测试框架说明；本卡未发现独立前沿风险评估或防护有效性章节。" } } ], "dateEvidence": [] }, { "id": "zai-glm-5-3-bf16-model-card", "sourceId": "official-zai-glm-5-3-bf16-model-card", "developerId": "zai", "title": "GLM-5.3-BF16 Model Card", "documentType": "model-card", "publicationDate": null, "updatedDate": null, "models": [ "GLM-5.3-BF16" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "Benchmark; Footnotes", "locator": "Original source → Benchmark; Footnotes", "protocolNames": [ "DeepSWE" ], "disposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established", "notes": { "en": "Latest linked card references the GLM-5 report; it does not independently establish the safety findings of GLM-5.2 external evaluations.", "zh": "最新链接卡引用GLM-5报告；不能据此建立外部GLM-5.2评估的安全结论。" } } ], "dateEvidence": [] }, { "id": "zai-glm-5-from-vibe-coding-to-agentic-engineering", "sourceId": "official-zai-glm-5-from-vibe-coding-to-agentic-engineering", "developerId": "zai", "title": "GLM-5: from Vibe Coding to Agentic Engineering", "documentType": "technical-report", "publicationDate": null, "updatedDate": null, "models": [ "GLM-5" ], "reviewLevel": "bounded-source-sections", "sections": [ { "heading": "3.4 General RL; agentic training", "locator": "Original source → 3.4 General RL; agentic training", "protocolNames": [], "disposition": "source-section-reviewed; specific-frontier-risk-evidence-not-established", "notes": { "en": "Reward-model robustness and training-time reward hacking are discussed; there is no dedicated catastrophic-risk assessment in the reviewed report.", "zh": "讨论奖励模型稳健性与训练奖励投机；所审报告没有独立灾难风险评估。" } } ], "dateEvidence": [ { "field": "publicationDate", "value": null, "source": "https://arxiv.org/abs/2602.15763 → Submitted on; this is arXiv submission, not a model release." } ] } ] }