{"schemaVersion":"nuwa-record-evidence-1","batchId":"research-2026-10-28","baseSnapshotId":"2026-10-27.3","publication":"published","dossier":{"id":"monitor-redteam","en":{"takeaway":"Monitor coverage must be measured independently of classifier accuracy; historic estimates and later patch claims are distinct.","question":"Can a monitor evaluate the traffic that actually reaches a deployed agent?","design":"The public report describes a three-week, high-access internal monitoring pilot. It separates traffic collection, monitor-system integrity and the response path. This reading covers the public findings and footnotes, not an internal audit package.","findings":["The pilot reported coverage and integrity problems. Approximately 10% unmonitored traffic was a historical estimate under incomplete logging; later remediation claims were not re-evaluated by this study."],"interpretation":"Monitor performance requires evidence about collection, independent judgment and response. Offline classifier accuracy does not establish the integrity of that chain.","limits":"The public pilot summary is neither a current vulnerability inventory nor a precise estimate for all deployed traffic.","connections":[],"decisionRelevance":"NUWA recommendation: Compare independently measured traffic coverage and monitor integrity with the assumptions made in an AI control or safety case."},"zh":{"takeaway":"高权限试点发现流量覆盖与系统完整性问题；约 10% 漏监只是历史估计，厂商修复说法未经本研究复测。","question":"部署监测能否覆盖实际智能体流量？","design":"公开报告回顾一个为期三周的高权限内部监测试点，区分流量是否被采集、监测组件是否可被修改，以及告警能否进入响应流程。这里审阅的是公开摘要及脚注，未取得完整内部审计材料。","findings":["试点识别到流量覆盖和监测系统完整性问题。约 10% 未监测流量是完整日志不足时的历史估计；本研究没有复测厂商之后宣称的修复。"],"interpretation":"监测效果需要一条完整的证据链：流量进入、独立判定、告警与响应。离线分类准确率不能替代对这条链的完整性检查。","limits":"仅公开试点摘要；不能当作当前漏洞清单或所有流量的精确漏检率。","connections":[],"decisionRelevance":"NUWA 建议：公开试点是否可支持更强的独立审计及持续对账？"},"assertionPolicy":{"findings":"source-finding","interpretation":"nuwa-interpretation","decisionRelevance":"recommendation"},"observations":[],"relatedWorkIds":["ai-control"],"supplementSourceId":"monitor-redteam","batchId":"research-2026-10-28","sourceRefs":[{"sourceId":"dr-20261028-monitor-redteam","url":"https://metr.org/blog/2026-10-22-red-teaming-openai-agent-monitoring/","title":"Red-Teaming OpenAI’s Internal Agent Monitoring Systems","locator":"Public findings and footnotes; text lines 00058","version":"Original resource captured 2026-10-28T07:24:41.534220+00:00; sha256:3f696499dad43c8695773e824cdad04269e74a9bec4b030a65710ecbf9657f47","artifactSha256":"3f696499dad43c8695773e824cdad04269e74a9bec4b030a65710ecbf9657f47","textSha256":"3e1942f21e90210c086bc5b4ade3d00048c70656d0d8e2bfa533e09531a3ce19","capturedAt":"2026-10-28T07:24:41.534220+00:00","resourceEdition":"Red-Teaming OpenAI's Internal Agent Monitoring Systems - METRSubstacktwitterBluesky"}],"publication":{"state":"published","publishedOn":"2026-10-29","authorizationReference":"user-authorized-global-index-publication-2026-10-29","reviewedCandidateId":"local-6efe31bc4a0c0f37"},"review":{"state":"published-note","sourceScope":"full-text","availableScope":null,"readingCoverage":"selected-original-sections","completePaperAudit":false,"independentEvidenceReview":"pending-human-review","bilingualEditorialReview":"pending-human-review","authorship":"model-assisted-research-note","preparedFingerprint":"5bdf4172a58d12ea4155d7bfbf69d7b44c04372df2ee2df8546ec3bc8f2e812b"},"supplementalSources":[{"id":"monitor-redteam","en":{"question":"Does intended monitoring coverage translate into actual oversight?","design":"A high-access pilot inspected internal infrastructure and staff accounts for a historical monitoring configuration.","findings":["Eligibility and delivered monitoring were different quantities; infrastructure gaps affected coverage.","Subsequent fixes were reported by the developer but not reevaluated."],"limits":"A partially disclosed historical pilot cannot establish present coverage or incident rates.","takeaway":["Eligibility and delivered monitoring were different quantities; infrastructure gaps affected coverage.","Subsequent fixes were reported by the developer but not reevaluated."]},"zh":{"question":"设计中的监测覆盖能否转化为实际监督？","design":"高访问权限试点检查内部设施及人员提供的材料，范围绑定历史监测配置。","findings":["具备监测资格与实际完成监测不是同一指标，基础设施缺口会影响覆盖。","厂商报告了后续修复，但研究者没有重新验证。"],"limits":"仅披露部分结果的历史试点，不能证明当前覆盖率或事件发生率。","takeaway":["具备监测资格与实际完成监测不是同一指标，基础设施缺口会影响覆盖。","厂商报告了后续修复，但研究者没有重新验证。"]},"observations":[{"id":"eligible-codex","rawToken":"99.2%","value":99.2,"unit":"%","locator":"L91","conditions":"Internal Codex subset of Responses API traffic; eligible for monitoring, not successful detection; system on/before July 21 2026","denominator":null,"interval":null,"status":"source-reported-unreviewed","use":"record-evidence-only","trendEligible":false,"rawValue":"99.2%","metricName":{"en":"eligible codex","zh":"eligible-codex"},"observationKind":"operational-coverage","sampleCounts":{"tasks":null,"runs":null,"attempts":null},"reportedUncertainty":null,"numerator":null,"publication":"published"},{"id":"eligible-agentic-range","rawToken":"75–94%","value":null,"unit":"%","locator":"L87-L91","conditions":"Approximate estimated range, not a confidence interval; agentic internal Responses API traffic from models at least as strong as GPT-5.5, for historical monitoring system on/before July 21 2026; excludes other internal inference services. Eligibility is not completed monitoring or successful detection.","denominator":null,"interval":null,"status":"source-reported-unreviewed","use":"record-evidence-only","trendEligible":false,"rawValue":"75–94%","metricName":{"en":"eligible agentic range","zh":"eligible-agentic-range"},"observationKind":"operational-coverage","sampleCounts":{"tasks":null,"runs":null,"attempts":null},"reportedUncertainty":null,"numerator":null,"publication":"published"},{"id":"compaction-unmonitored","rawValue":"on the order of 10%","rawToken":"on the order of 10%","value":10,"unit":"%","metricName":{"en":"Eligible Codex traffic missed due to compaction","zh":"压缩事件导致未监测的合格 Codex 流量"},"locator":"L93","conditions":"May–July 2026; historical Codex traffic eligible for monitoring; approximate estimate; substantial uncertainty","observationKind":"operational-coverage","system":null,"denominator":null,"numerator":null,"qualifier":"approximate","sampleCounts":{"tasks":null,"runs":null,"attempts":null},"reportedUncertainty":null,"status":"source-reported-unreviewed","use":"record-evidence-only","trendEligible":false,"sourceUrl":"https://metr.org/blog/2026-10-22-red-teaming-openai-agent-monitoring/","sourceSha256":"3f696499dad43c8695773e824cdad04269e74a9bec4b030a65710ecbf9657f47","publication":"published"},{"id":"monitor-errors","rawValue":"less than 0.2%","rawToken":"less than 0.2%","value":0.2,"unit":"%","metricName":{"en":"Monitor request error rate upper bound","zh":"监测请求错误率上界"},"locator":"L96","conditions":"Information shown to researchers at exercise time, historical system on/before 2026-07-21; request denominator not disclosed","observationKind":"operational-coverage","system":null,"denominator":null,"numerator":null,"qualifier":"less-than","sampleCounts":{"tasks":null,"runs":null,"attempts":null},"reportedUncertainty":null,"status":"source-reported-unreviewed","use":"record-evidence-only","trendEligible":false,"sourceUrl":"https://metr.org/blog/2026-10-22-red-teaming-openai-agent-monitoring/","sourceSha256":"3f696499dad43c8695773e824cdad04269e74a9bec4b030a65710ecbf9657f47","publication":"published"},{"id":"zero-load-sheds","rawValue":"zero","rawToken":"zero","value":0,"unit":"count","metricName":{"en":"Load-shed events in observed window","zh":"所报告窗口内负载丢弃次数"},"locator":"L96","conditions":"One 24-hour period during exercise; not a long-run failure-rate estimate","observationKind":"operational-coverage","system":null,"denominator":null,"numerator":null,"qualifier":"exact","sampleCounts":{"tasks":null,"runs":null,"attempts":null},"reportedUncertainty":null,"status":"source-reported-unreviewed","use":"record-evidence-only","trendEligible":false,"sourceUrl":"https://metr.org/blog/2026-10-22-red-teaming-openai-agent-monitoring/","sourceSha256":"3f696499dad43c8695773e824cdad04269e74a9bec4b030a65710ecbf9657f47","publication":"published"}],"unitInventory":[{"id":"metr-openai-monitoring-section-1","kind":"section","title":"Featured research","locator":"L174","disposition":"not-applicable","reason":"Navigation or related-content heading outside captured article body","observationIds":[]},{"id":"metr-openai-monitoring-section-2","kind":"section","title":"Measuring the Self-Reported Impact of Early-2026 AI on Technical Worker Productivity","locator":"L176","disposition":"pending-review","reason":"Heading discovered from original capture; full unit not yet audited","observationIds":[]},{"id":"metr-openai-monitoring-section-3","kind":"section","title":"Early Work on Monitorability Evaluations","locator":"L179","disposition":"pending-review","reason":"Heading discovered from original capture; full unit not yet audited","observationIds":[]},{"id":"metr-openai-monitoring-section-4","kind":"section","title":"How Does Time Horizon Vary Across Domains?","locator":"L182","disposition":"pending-review","reason":"Heading discovered from original capture; full unit not yet audited","observationIds":[]},{"id":"metr-openai-monitoring-image-1","kind":"image","title":"METR Logo","locator":"HTML img 1","url":"https://metr.org/assets/images/logo/logo.svg","disposition":"pending-relevance-and-extraction","reason":"HTML image inventory includes publication assets and site graphics; chart coordinates not digitized or guessed"},{"id":"metr-openai-monitoring-image-2","kind":"image","title":"METR Logo","locator":"HTML img 2","url":"https://metr.org/assets/images/logo/logo.svg","disposition":"pending-relevance-and-extraction","reason":"HTML image inventory includes publication assets and site graphics; chart coordinates not digitized or guessed"},{"id":"metr-openai-monitoring-image-3","kind":"image","title":"METR Logo","locator":"HTML img 3","url":"https://metr.org/assets/images/gen/home/evals-logo-slideable.svg","disposition":"pending-relevance-and-extraction","reason":"HTML image inventory includes publication assets and site graphics; chart coordinates not digitized or guessed"},{"id":"metr-openai-monitoring-image-4","kind":"image","title":"Measuring the Self-Reported Impact of Early-2026 AI on Technical Worker Productivity","locator":"HTML img 4","url":"https://metr.org/assets/images/ai-usage-survey/headline.png","disposition":"pending-relevance-and-extraction","reason":"HTML image inventory includes publication assets and site graphics; chart coordinates not digitized or guessed"},{"id":"metr-openai-monitoring-image-5","kind":"image","title":"Early Work on Monitorability Evaluations","locator":"HTML img 5","url":"https://metr.org/assets/images/monitorability-post/time_horizon_ratio_non_visible_reasoning.png","disposition":"pending-relevance-and-extraction","reason":"HTML image inventory includes publication assets and site graphics; chart coordinates not digitized or guessed"},{"id":"metr-openai-monitoring-image-6","kind":"image","title":"How Does Time Horizon Vary Across Domains?","locator":"HTML img 6","url":"https://metr.org/assets/images/time-horizon-domains/time-horizons-increasing.png","disposition":"pending-relevance-and-extraction","reason":"HTML image inventory includes publication assets and site graphics; chart coordinates not digitized or guessed"},{"id":"metr-openai-monitoring-image-7","kind":"image","title":"METR Logo","locator":"HTML img 7","url":"https://metr.org/assets/images/gen/home/evals-logo-slideable.svg","disposition":"pending-relevance-and-extraction","reason":"HTML image inventory includes publication assets and site graphics; chart coordinates not digitized or guessed"},{"id":"metr-openai-monitoring-value-eligible-codex","kind":"numeric-statement","locator":"L91","disposition":"extracted","observationIds":["eligible-codex"],"reason":"Transcribed original value; independent human review remains pending"},{"id":"metr-openai-monitoring-value-eligible-agentic-range","kind":"numeric-statement","locator":"L87-L91","disposition":"extracted","observationIds":["eligible-agentic-range"],"reason":"Transcribed original value; independent human review remains pending"},{"id":"metr-openai-monitoring-value-compaction-unmonitored","kind":"numeric-statement","locator":"L93","disposition":"extracted","observationIds":["compaction-unmonitored"],"reason":"Transcribed original value; independent human review remains pending"},{"id":"metr-openai-monitoring-value-monitor-errors","kind":"numeric-statement","locator":"L96","disposition":"extracted","observationIds":["monitor-errors"],"reason":"Transcribed original value; independent human review remains pending"},{"id":"metr-openai-monitoring-value-zero-load-sheds","kind":"numeric-statement","locator":"L96","disposition":"extracted","observationIds":["zero-load-sheds"],"reason":"Transcribed original value; independent human review remains pending"}],"sourceScope":"Selected substantive passages of original institutional article; linked full paper, data and figures not independently audited","sourceRefs":[{"sourceId":"dr-20261028-monitor-redteam","url":"https://metr.org/blog/2026-10-22-red-teaming-openai-agent-monitoring/","version":"3f696499dad43c8695773e824cdad04269e74a9bec4b030a65710ecbf9657f47","locator":"L69-L118; L125-L141"}],"publication":{"state":"published","publishedOn":"2026-10-29","authorizationReference":"user-authorized-global-index-publication-2026-10-29","reviewedCandidateId":"local-6efe31bc4a0c0f37"},"review":{"state":"published-note","sourceScope":"selected-sections","independentEvidenceReview":"pending-human-review","bilingualEditorialReview":"pending-human-review","preparedFingerprint":"c0fe30bdf75be15e332af9b3eae6d9b1c5160e48b561c1589924fcd52e45356e"}}]},"update":null}