{"schemaVersion":"nuwa-current-record-1","version":"2026-10-29.1","publication":"published","work":{"id":"transect","title":"Transect: Retaining Observability for Long-Horizon LLM Agent Evaluations","aliases":[],"authors":[],"primaryFunction":"evaluation","secondaryFunctions":[],"kind":"method","sourceId":"transect-publication","attributionSourceId":"transect-publication","originalUrl":"https://www.aisi.gov.uk/research/transect-retaining-observability-for-long-horizon-llm-agent-evaluations","publicationDate":"2026-10-07","updatedOn":"2026-10-25","areas":[],"scopeNote":{"en":"Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.","zh":"风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。"},"protocolIds":[],"contributions":[{"organizationId":"aisi","role":"publisher","sourceId":"transect-publication","locator":"Official publication masthead and article byline"}],"summary":{"en":"Structures transcript analysis for long-horizon agent evaluations so interpretations can be checked against retained evidence.","zh":"组织长程智能体评测轨迹的分析，使解释能够对照留存证据核查。"},"note":{"question":null,"method":null,"finding":null,"supports":null,"limits":null,"connections":null},"resources":[{"label":"Read the full paper","url":"https://arxiv.org/abs/2610.08364"}],"noteStatus":{"question":"limited","method":"limited","finding":"limited","supports":"limited","limits":"limited","connections":"limited"},"noteAvailability":{"en":"A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.","zh":"实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。"},"noteSourceLocator":"No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence.","tags":["risk-monitoring"]},"reading":{"id":"transect","title":"Transect: Retaining Observability for Long-Horizon LLM Agent Evaluations","en":{"question":"How can evaluators preserve observability and auditability in very long agent runs?","design":"Transect separates reusable task/behavior specifications from judge settings and aligns events, token use, subagents and labels on a source-linked timeline.","findings":["The case study analyzes an AI R&D evaluation producing almost thirteen million tokens.","The conclusion distinguishes reliability diagnostics from validity, which requires separate evidence such as independent expert annotations."],"interpretation":"NUWA interpretation: navigable provenance helps inspect a behavioral claim without itself proving the label correct.","limits":"LLM scanners can overstate confidence and obscure uncertainty; the case study is not a validation across all scientific work.","decisionRelevance":"Model recommendation: preserve source-turn anchors, judge configuration and disagreement diagnostics with exported analyses.","takeaway":"NUWA interpretation: navigable provenance helps inspect a behavioral claim without itself proving the label correct.","connections":[],"scoring":null,"conditions":null,"statistics":null,"usage":"Model recommendation: preserve source-turn anchors, judge configuration and disagreement diagnostics with exported analyses."},"zh":{"question":"长智能体运行如何保留可观察性与可审计性？","design":"Transect将可复用任务行为定义与判官设置分开，在源链接时间线上对齐事件、令牌、子智能体和标签。","findings":["个案分析近1,300万令牌的AI研发评测。","结论区分可靠性诊断与效度，后者需独立专家标注等额外证据。"],"interpretation":"NUWA 解读：可导航来源有助检查行为结论，但不能证明标签正确。","limits":"模型扫描器可能夸大信心或隐藏不确定性，单个研究例不验证所有科学工作。","decisionRelevance":"模型建议：导出分析时保留源轮次、判官配置及分歧诊断。","takeaway":"NUWA 解读：可导航来源有助检查行为结论，但不能证明标签正确。","connections":[],"scoring":null,"conditions":null,"statistics":"个案分析近1,300万令牌的AI研发评测。","usage":"模型建议：导出分析时保留源轮次、判官配置及分歧诊断。"},"assertionPolicy":{"question":"source-research-question","design":"source-finding","findings":"source-finding","interpretation":"nuwa-interpretation","limits":"source-limit-or-explicit-editorial-scope-limit","connections":"source-supported-relationship-only","decisionRelevance":"model-recommendation-not-source-conclusion"},"sourceRefs":[{"sourceId":"transect-publication","url":"https://arxiv.org/html/2610.08364","locator":"Abstract","version":"Selected original resource https://arxiv.org/html/2610.08364; sha256:15cf35ee7c1d3115d5dac5322b3352f66d9b2564d843dfac2b42e43cf09c6480; archived source capture; exact capture time not independently established","artifactSha256":"15cf35ee7c1d3115d5dac5322b3352f66d9b2564d843dfac2b42e43cf09c6480"},{"sourceId":"transect-publication","url":"https://arxiv.org/html/2610.08364","locator":"§2.2 Design principles","version":"Selected original resource https://arxiv.org/html/2610.08364; sha256:15cf35ee7c1d3115d5dac5322b3352f66d9b2564d843dfac2b42e43cf09c6480; archived source capture; exact capture time not independently established","artifactSha256":"15cf35ee7c1d3115d5dac5322b3352f66d9b2564d843dfac2b42e43cf09c6480"},{"sourceId":"transect-publication","url":"https://arxiv.org/html/2610.08364","locator":"§3 Case Study: Reading an AI R&D Evaluation with Transect","version":"Selected original resource https://arxiv.org/html/2610.08364; sha256:15cf35ee7c1d3115d5dac5322b3352f66d9b2564d843dfac2b42e43cf09c6480; archived source capture; exact capture time not independently established","artifactSha256":"15cf35ee7c1d3115d5dac5322b3352f66d9b2564d843dfac2b42e43cf09c6480"},{"sourceId":"transect-publication","url":"https://arxiv.org/html/2610.08364","locator":"§4 Conclusion","version":"Selected original resource https://arxiv.org/html/2610.08364; sha256:15cf35ee7c1d3115d5dac5322b3352f66d9b2564d843dfac2b42e43cf09c6480; archived source capture; exact capture time not independently established","artifactSha256":"15cf35ee7c1d3115d5dac5322b3352f66d9b2564d843dfac2b42e43cf09c6480"}],"publication":{"state":"published","publishedOn":"2026-10-29","authorizationReference":"user-authorized-global-index-publication-2026-10-29","reviewedCandidateId":"local-6efe31bc4a0c0f37"},"review":{"state":"published-note","sourceScope":"full-text","availableScope":null,"readingCoverage":"selected-source-sections; not a complete section-by-section paper audit","completePaperAudit":false,"independentEvidenceReview":"pending-human-review","bilingualEditorialReview":"pending-human-review","authorship":"model-assisted-research-note","preparedFingerprint":"d36452daa3acb2d5b83a5c22208136138850a4a185573ebb6aebe1a4ba2f2a97"}},"sources":[{"id":"transect-publication","title":"Transect: Retaining Observability for Long-Horizon LLM Agent Evaluations","url":"https://www.aisi.gov.uk/research/transect-retaining-observability-for-long-horizon-llm-agent-evaluations","version":"Source capture 2026-10-25","publicationDate":"2026-10-07","datePrecision":"day","type":"research-publication","verifiedOn":"2026-10-25","documentId":"https://www.aisi.gov.uk/research/transect-retaining-observability-for-long-horizon-llm-agent-evaluations"}]}