{"schemaVersion":"nuwa-current-record-1","version":"2026-10-29.1","publication":"published","work":{"id":"fortress","title":"FORTRESS: Frontier Risk Evaluation for National Security and Public Safety","aliases":["FORTRESS"],"authors":["Christina Q. Knight","Kaustubh Deshpande","Ved Sirdeshmukh","Meher Mankikar","Scale Red Team","SEAL Research Team","Julian Michael"],"primaryFunction":"evaluation","secondaryFunctions":[],"kind":"benchmark","sourceId":"fortress-publication","attributionSourceId":"fortress-publication-author-source","originalUrl":"https://arxiv.org/abs/2506.14922","publicationDate":"2025-06-17","updatedOn":"2026-10-25","areas":["cbrn","cybersecurity"],"scopeNote":{"en":"The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.","zh":"主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。"},"protocolIds":["fortress"],"contributions":[{"organizationId":"scale","role":"research","sourceId":"fortress-publication-author-source","locator":"Author affiliations on the original paper title page"}],"summary":{"en":"Evaluates safeguards on national-security and public-safety tasks, separating hazardous requests from benign-use utility.","zh":"在国家安全与公共安全任务上评测防护，区分高危请求与正常用途的有用性。"},"note":{"question":null,"method":null,"finding":null,"supports":null,"limits":null,"connections":null},"resources":[],"noteStatus":{"question":"limited","method":"limited","finding":"limited","supports":"limited","limits":"limited","connections":"limited"},"noteAvailability":{"en":"A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.","zh":"实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。"},"noteSourceLocator":"No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence.","tags":[]},"reading":{"id":"fortress","title":"FORTRESS: Frontier Risk Evaluation for National Security and Public Safety","en":{"question":"How do safeguards balance national/public-safety risk restriction against benign access?","design":"500 expert-crafted adversarial prompts each have a benign counterpart and a 4–7-item binary rubric, spanning three broad domains.","findings":["Claude 3.5 Sonnet’s reported ARS/ORS were 14.09/21.80, versus Gemini 2.5 Pro’s 66.29/1.40.","These are rubric-derived risk and over-refusal scores, not harm probabilities."],"interpretation":"NUWA interpretation: restriction and benign usefulness form two axes that a single refusal rate would hide.","limits":"Static single-turn prompts, limited domain expertise and sparse niche coverage constrain validity (§6); judge correctness requires separate audit.","decisionRelevance":"Model recommendation: retain ARS/ORS together and distinguish the original paper’s estimates from subsequent grading audits.","takeaway":"NUWA interpretation: restriction and benign usefulness form two axes that a single refusal rate would hide.","connections":[],"scoring":"Average Risk Score and Over-Refusal Score from per-item binary rubrics; not probabilities of realized harm.","conditions":"500 adversarial/benign prompt pairs, single-turn; judge implementation subject to separate audit.","statistics":"Claude 3.5 Sonnet’s reported ARS/ORS were 14.09/21.80, versus Gemini 2.5 Pro’s 66.29/1.40.","usage":"Model recommendation: retain ARS/ORS together and distinguish the original paper’s estimates from subsequent grading audits."},"zh":{"question":"防护如何在国家公共安全限制与良性访问间权衡？","design":"500项专家对抗提示各配良性版本和4–7项二元量表，覆盖三个大领域。","findings":["Claude 3.5 Sonnet所报ARS/ORS为14.09/21.80，Gemini 2.5 Pro为66.29/1.40。","这是量表风险与过拒分数，不是伤害概率。"],"interpretation":"NUWA 解读：限制与良性效用是两个维度，单一拒绝率会掩盖。","limits":"固定单轮提示、专业深度及小领域覆盖有限（§6），判官正确性还需单独审计。","decisionRelevance":"模型建议：同时保留ARS/ORS，并区分原论文估计与后续判分审计。","takeaway":"NUWA 解读：限制与良性效用是两个维度，单一拒绝率会掩盖。","connections":[],"scoring":"逐题二元量表的平均风险分及过拒分，不是实际伤害概率。","conditions":"500对对抗良性提示，单轮，判分实现另须审计。","statistics":"Claude 3.5 Sonnet所报ARS/ORS为14.09/21.80，Gemini 2.5 Pro为66.29/1.40。","usage":"模型建议：同时保留ARS/ORS，并区分原论文估计与后续判分审计。"},"assertionPolicy":{"question":"source-research-question","design":"source-finding","findings":"source-finding","interpretation":"nuwa-interpretation","limits":"source-limit-or-explicit-editorial-scope-limit","connections":"source-supported-relationship-only","decisionRelevance":"model-recommendation-not-source-conclusion"},"sourceRefs":[{"sourceId":"fortress-publication-author-source","url":"https://arxiv.org/html/2506.14922","locator":"Abstract","version":"Selected original resource https://arxiv.org/html/2506.14922; sha256:6d8ae92cd9c9950fafbc23990e3a18f67cf4f527869648b8351d21e52bc62efc; archived source capture; exact capture time not independently established","artifactSha256":"6d8ae92cd9c9950fafbc23990e3a18f67cf4f527869648b8351d21e52bc62efc"},{"sourceId":"fortress-publication-author-source","url":"https://arxiv.org/html/2506.14922","locator":"§5.1 Main Results","version":"Selected original resource https://arxiv.org/html/2506.14922; sha256:6d8ae92cd9c9950fafbc23990e3a18f67cf4f527869648b8351d21e52bc62efc; archived source capture; exact capture time not independently established","artifactSha256":"6d8ae92cd9c9950fafbc23990e3a18f67cf4f527869648b8351d21e52bc62efc"},{"sourceId":"fortress-publication-author-source","url":"https://arxiv.org/html/2506.14922","locator":"§6 Limitations","version":"Selected original resource https://arxiv.org/html/2506.14922; sha256:6d8ae92cd9c9950fafbc23990e3a18f67cf4f527869648b8351d21e52bc62efc; archived source capture; exact capture time not independently established","artifactSha256":"6d8ae92cd9c9950fafbc23990e3a18f67cf4f527869648b8351d21e52bc62efc"}],"publication":{"state":"published","publishedOn":"2026-10-29","authorizationReference":"user-authorized-global-index-publication-2026-10-29","reviewedCandidateId":"local-6efe31bc4a0c0f37"},"review":{"state":"published-note","sourceScope":"full-text","availableScope":null,"readingCoverage":"selected-source-sections; not a complete section-by-section paper audit","completePaperAudit":false,"independentEvidenceReview":"pending-human-review","bilingualEditorialReview":"pending-human-review","authorship":"model-assisted-research-note","preparedFingerprint":"c31d9d37c48e2c41bd292759a346ca7ee8f791b1bd62356e59ec3d93cb389471"}},"sources":[{"id":"fortress-publication","title":"FORTRESS: Frontier Risk Evaluation for National Security and Public Safety","url":"https://arxiv.org/abs/2506.14922","version":"arXiv:2506.14922v2","publicationDate":"2025-06-17","datePrecision":"day","type":"paper","verifiedOn":"2026-10-25","documentId":"arxiv:2506.14922","authors":["Christina Q. Knight ∗","Kaustubh Deshpande ⋄","Ved Sirdeshmukh ⋄","Meher Mankikar","Scale Red Team","SEAL Research Team","and Julian Michael"],"aliases":["https://doi.org/10.48550/arXiv.2506.14922","https://arxiv.org/abs/2506.14922","https://arxiv.org/pdf/2506.14922"]}]}