{"schemaVersion":"nuwa-current-record-1","version":"2026-10-29.1","publication":"published","work":{"id":"caisi-deepseek-v4-report","title":"CAISI Evaluation of DeepSeek V4 Pro","aliases":[],"authors":[],"primaryFunction":"evaluation","secondaryFunctions":[],"kind":"technical-report","sourceId":"caisi-deepseek-v4-source","attributionSourceId":"caisi-deepseek-v4-source","originalUrl":"https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro","publicationDate":"2026-05-01","updatedOn":"2026-10-25","areas":["autonomy","deception","cybersecurity","cbrn","disinformation"],"scopeNote":{"en":"Reported results retain their original evaluator, configuration and measurement scope. NUWA organizes and interprets the evidence; it has not independently reproduced these experiments.","zh":"结果保留原始执行方、配置与测量范围。NUWA 整理和解读证据，未独立复现这些实验。"},"protocolIds":["ctf-archive-diamond","portbench"],"contributions":[{"organizationId":"nist","role":"research","sourceId":"caisi-deepseek-v4-source","locator":"Report masthead and stated evaluation methodology"}],"summary":{"en":"The assessment reports benchmark-specific performance and an IRT aggregate; the latter is not a frontier-risk score.","zh":"评估报告各基准表现和 IRT 汇总；该汇总不是前沿风险分数。"},"note":{"question":{"en":"How does DeepSeek V4 Pro perform across CAISI’s selected capability tasks?","zh":"DeepSeek V4 Pro 在 CAISI 选定能力任务上表现如何？"},"method":{"en":"Nine task families, model-specific reasoning settings and weighted-token budgets; CTF results for DeepSeek include IRT imputation.","zh":"九类任务、模型特定推理设置与加权 Token 预算；DeepSeek CTF 结果包含 IRT 插补。"},"finding":{"en":"The assessment reports benchmark-specific performance and an IRT aggregate; the latter is not a frontier-risk score.","zh":"评估报告各基准表现和 IRT 汇总；该汇总不是前沿风险分数。"},"supports":null,"limits":{"en":"Controlled tasks, elicitation, benchmark familiarity and configuration differences constrain generalization to deployment.","zh":"受控任务、诱发方式、基准熟悉程度和配置差异限制对部署的外推。"},"connections":null},"resources":[],"tags":[]},"reading":{"id":"caisi-deepseek-v4-report","title":"CAISI Evaluation of DeepSeek V4 Pro","en":{"question":"How does an independently selected evaluation suite change the picture of DeepSeek V4 Pro?","design":"Nine benchmarks span five domains, with two held-out datasets; a 1PL IRT fit weights benchmarks within domains and domains equally.","findings":["The report estimates an approximately eight-month aggregate capability lag in its chosen suite.","It reproduces GPQA-Diamond to check inference/configuration and notes prompt/scaffold/budget differences for SWE-bench."],"interpretation":"NUWA interpretation: the lag is a model-based historical summary, not uniform underperformance on every task.","limits":"CTF results include IRT imputation and PortBench continuous scoring; cost analysis applies only to seven eligible benchmarks and shared solved tasks.","decisionRelevance":"Model recommendation: retain benchmark-level evidence, IRT assumptions and exclusions with any aggregate claim.","takeaway":"NUWA interpretation: the lag is a model-based historical summary, not uniform underperformance on every task.","connections":[],"scoring":null,"conditions":null,"statistics":null,"usage":"Model recommendation: retain benchmark-level evidence, IRT assumptions and exclusions with any aggregate claim."},"zh":{"question":"独立选择的评测集合如何改变DeepSeek V4 Pro能力图景？","design":"九基准覆盖五领域含两个保留集，1PL IRT按领域内基准及领域等权拟合。","findings":["报告在所选集合估计综合能力约落后八个月。","复现GPQA-Diamond检查推理配置，并注明SWE-bench提示框架预算差异。"],"interpretation":"NUWA 解读：落后时间是模型化历史总结，不代表每任务均落后。","limits":"CTF含IRT推补，PortBench连续计分；成本仅覆盖七个合格基准及共同解出任务。","decisionRelevance":"模型建议：综合结论旁保留逐基准证据、IRT假设与排除规则。","takeaway":"NUWA 解读：落后时间是模型化历史总结，不代表每任务均落后。","connections":[],"scoring":null,"conditions":null,"statistics":null,"usage":"模型建议：综合结论旁保留逐基准证据、IRT假设与排除规则。"},"assertionPolicy":{"question":"source-research-question","design":"source-finding","findings":"source-finding","interpretation":"nuwa-interpretation","limits":"source-limit-or-explicit-editorial-scope-limit","connections":"source-supported-relationship-only","decisionRelevance":"model-recommendation-not-source-conclusion"},"sourceRefs":[{"sourceId":"caisi-deepseek-v4-source","url":"https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro","locator":"Capability Results","version":"Selected original resource https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro; sha256:37f75aab5640e04d198b94a0ca57704b06d7823caeeb05c4280c8fc461b00623; archived source capture; exact capture time not independently established","artifactSha256":"37f75aab5640e04d198b94a0ca57704b06d7823caeeb05c4280c8fc461b00623"},{"sourceId":"caisi-deepseek-v4-source","url":"https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro","locator":"Benchmarks","version":"Selected original resource https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro; sha256:37f75aab5640e04d198b94a0ca57704b06d7823caeeb05c4280c8fc461b00623; archived source capture; exact capture time not independently established","artifactSha256":"37f75aab5640e04d198b94a0ca57704b06d7823caeeb05c4280c8fc461b00623"},{"sourceId":"caisi-deepseek-v4-source","url":"https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro","locator":"Capability Lag Measurement","version":"Selected original resource https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro; sha256:37f75aab5640e04d198b94a0ca57704b06d7823caeeb05c4280c8fc461b00623; archived source capture; exact capture time not independently established","artifactSha256":"37f75aab5640e04d198b94a0ca57704b06d7823caeeb05c4280c8fc461b00623"},{"sourceId":"caisi-deepseek-v4-source","url":"https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro","locator":"Comparison of DeepSeek and CAISI Evaluations","version":"Selected original resource https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro; sha256:37f75aab5640e04d198b94a0ca57704b06d7823caeeb05c4280c8fc461b00623; archived source capture; exact capture time not independently established","artifactSha256":"37f75aab5640e04d198b94a0ca57704b06d7823caeeb05c4280c8fc461b00623"},{"sourceId":"caisi-deepseek-v4-source","url":"https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro","locator":"DeepSeek V4 Costs Less Than Other Models Of Similar Capability","version":"Selected original resource https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro; sha256:37f75aab5640e04d198b94a0ca57704b06d7823caeeb05c4280c8fc461b00623; archived source capture; exact capture time not independently established","artifactSha256":"37f75aab5640e04d198b94a0ca57704b06d7823caeeb05c4280c8fc461b00623"},{"sourceId":"caisi-deepseek-v4-source","url":"https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro","locator":"Appendix","version":"Selected original resource https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro; sha256:37f75aab5640e04d198b94a0ca57704b06d7823caeeb05c4280c8fc461b00623; archived source capture; exact capture time not independently established","artifactSha256":"37f75aab5640e04d198b94a0ca57704b06d7823caeeb05c4280c8fc461b00623"}],"publication":{"state":"published","publishedOn":"2026-10-29","authorizationReference":"user-authorized-global-index-publication-2026-10-29","reviewedCandidateId":"local-6efe31bc4a0c0f37"},"review":{"state":"published-note","sourceScope":"full-text","availableScope":null,"readingCoverage":"selected-source-sections; not a complete section-by-section paper audit","completePaperAudit":false,"independentEvidenceReview":"pending-human-review","bilingualEditorialReview":"pending-human-review","authorship":"model-assisted-research-note","preparedFingerprint":"4deaf7b3704ed59a06b5b1fa89e7be436a13a2cc9fcf030aaba60a40c65c054d"}},"sources":[{"id":"caisi-deepseek-v4-source","title":"CAISI Evaluation of DeepSeek V4 Pro","url":"https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro","version":"1 May 2026","publicationDate":"2026-05-01","datePrecision":"day","type":"original-report","verifiedOn":"2026-10-25","authors":[],"documentId":"https://www.nist.gov/news-events/news/2026/05/caisi-evaluation-deepseek-v4-pro"}]}