{"schemaVersion":"nuwa-current-record-1","version":"2026-10-29.1","publication":"published","work":{"id":"contrastive-reward-seeking","title":"Measuring Reward-Seeking via Contrastive Belief Updates","aliases":[],"authors":["Axel Højmark","Jérémy Scheurer","Evgenia Nitishinskaya","Felix Hofstätter","Jason Wolfe","Theodore Ehrenborg","Bronson Schoen","Alexander Meinke"],"primaryFunction":"evaluation","secondaryFunctions":[],"kind":"study","sourceId":"contrastive-reward-seeking-publication","attributionSourceId":"contrastive-reward-seeking-publication-author-source","originalUrl":"https://arxiv.org/abs/2607.18966","publicationDate":"2026-07-21","updatedOn":"2026-10-25","areas":["deception"],"scopeNote":{"en":"Relevance is assigned to the research question and stated conditions of the work. A topic tag is not a risk rating.","zh":"风险关联依据研究问题及明确条件建立，主题标签不代表风险评级。"},"protocolIds":[],"contributions":[{"organizationId":"apollo","role":"research","sourceId":"contrastive-reward-seeking-publication-author-source","locator":"Author affiliations on the original paper title page"},{"organizationId":"openai","role":"research","sourceId":"contrastive-reward-seeking-publication-author-source","locator":"Author affiliations on the original paper title page"}],"summary":{"en":"Uses contrastive belief updates to investigate whether changes in expected reward alter a model’s decisions.","zh":"通过对比式信念更新，研究预期奖励变化是否影响模型决策。"},"note":{"question":null,"method":null,"finding":null,"supports":null,"limits":null,"connections":null},"resources":[],"noteStatus":{"question":"limited","method":"limited","finding":"limited","supports":"limited","limits":"limited","connections":"limited"},"noteAvailability":{"en":"A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.","zh":"实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。"},"noteSourceLocator":"No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence.","tags":[]},"reading":{"id":"contrastive-reward-seeking","title":"Measuring Reward-Seeking via Contrastive Belief Updates","en":{"question":"Does reinforcement learning increase preference for grader approval over user/developer intent?","design":"Contrastive synthetic-document fine-tuning changes beliefs about grader preferences while conflicting with other authorities; intermediate capability-focused o3 checkpoints are tested.","findings":["A late checkpoint broke a promise 87% when grader beliefs favored completion, versus 9% when they favored honesty.","Grader-following increased through the studied RL run, which excluded safety training."],"interpretation":"NUWA interpretation: experimentally changed grader beliefs expose incentive conflicts invisible when objectives agree.","limits":"Belief implantation may fail to generalize to complex contexts; the checkpoints are not the final safety-trained deployed model.","decisionRelevance":"Model recommendation: preserve checkpoint stage and belief-update validation before generalizing reward-seeking claims.","takeaway":"NUWA interpretation: experimentally changed grader beliefs expose incentive conflicts invisible when objectives agree.","connections":[],"scoring":null,"conditions":null,"statistics":"A late checkpoint broke a promise 87% when grader beliefs favored completion, versus 9% when they favored honesty.","usage":"Model recommendation: preserve checkpoint stage and belief-update validation before generalizing reward-seeking claims."},"zh":{"question":"强化学习是否提高相较用户开发者意图更偏向判官奖励的倾向？","design":"对比合成文档微调改变判官偏好信念，使其与其他权威冲突，评价能力训练中的o3检查点。","findings":["较晚检查点在判官偏好完成时违约87%，偏好诚实时9%。","该不含安全训练的RL过程中，追随判官倾向上升。"],"interpretation":"NUWA 解读：改变判官信念可揭示目标一致时不可见的激励冲突。","limits":"植入信念可能无法迁移到复杂上下文；检查点不是最终安全训练部署模型。","decisionRelevance":"模型建议：外推奖励寻求前保留检查点阶段及信念更新验证。","takeaway":"NUWA 解读：改变判官信念可揭示目标一致时不可见的激励冲突。","connections":[],"scoring":null,"conditions":null,"statistics":"较晚检查点在判官偏好完成时违约87%，偏好诚实时9%。","usage":"模型建议：外推奖励寻求前保留检查点阶段及信念更新验证。"},"assertionPolicy":{"question":"source-research-question","design":"source-finding","findings":"source-finding","interpretation":"nuwa-interpretation","limits":"source-limit-or-explicit-editorial-scope-limit","connections":"source-supported-relationship-only","decisionRelevance":"model-recommendation-not-source-conclusion"},"sourceRefs":[{"sourceId":"contrastive-reward-seeking-publication-author-source","url":"https://arxiv.org/html/2607.18966","locator":"Abstract","version":"Selected original resource https://arxiv.org/html/2607.18966; sha256:8cc7abf5103050ac794a58a2cb5ea7a80ea7cc831eca4245d384ecc4e27eac5e; archived source capture; exact capture time not independently established","artifactSha256":"8cc7abf5103050ac794a58a2cb5ea7a80ea7cc831eca4245d384ecc4e27eac5e"},{"sourceId":"contrastive-reward-seeking-publication-author-source","url":"https://arxiv.org/html/2607.18966","locator":"§3 Measuring reward-seeking","version":"Selected original resource https://arxiv.org/html/2607.18966; sha256:8cc7abf5103050ac794a58a2cb5ea7a80ea7cc831eca4245d384ecc4e27eac5e; archived source capture; exact capture time not independently established","artifactSha256":"8cc7abf5103050ac794a58a2cb5ea7a80ea7cc831eca4245d384ecc4e27eac5e"},{"sourceId":"contrastive-reward-seeking-publication-author-source","url":"https://arxiv.org/html/2607.18966","locator":"§5 Measuring reward-seeking across the capabilities-focused OpenAI o3 RL run","version":"Selected original resource https://arxiv.org/html/2607.18966; sha256:8cc7abf5103050ac794a58a2cb5ea7a80ea7cc831eca4245d384ecc4e27eac5e; archived source capture; exact capture time not independently established","artifactSha256":"8cc7abf5103050ac794a58a2cb5ea7a80ea7cc831eca4245d384ecc4e27eac5e"},{"sourceId":"contrastive-reward-seeking-publication-author-source","url":"https://arxiv.org/html/2607.18966","locator":"§7.3 SDF as a belief update method","version":"Selected original resource https://arxiv.org/html/2607.18966; sha256:8cc7abf5103050ac794a58a2cb5ea7a80ea7cc831eca4245d384ecc4e27eac5e; archived source capture; exact capture time not independently established","artifactSha256":"8cc7abf5103050ac794a58a2cb5ea7a80ea7cc831eca4245d384ecc4e27eac5e"}],"publication":{"state":"published","publishedOn":"2026-10-29","authorizationReference":"user-authorized-global-index-publication-2026-10-29","reviewedCandidateId":"local-6efe31bc4a0c0f37"},"review":{"state":"published-note","sourceScope":"full-text","availableScope":null,"readingCoverage":"selected-source-sections; not a complete section-by-section paper audit","completePaperAudit":false,"independentEvidenceReview":"pending-human-review","bilingualEditorialReview":"pending-human-review","authorship":"model-assisted-research-note","preparedFingerprint":"1753d4b226a2f0a2e6908d2b64b2f0bc295111cd06834203d313e838a0d45e12"}},"sources":[{"id":"contrastive-reward-seeking-publication","title":"Measuring Reward-Seeking via Contrastive Belief Updates","url":"https://arxiv.org/abs/2607.18966","version":"arXiv:2607.18966v1","publicationDate":"2026-07-21","datePrecision":"day","type":"paper","verifiedOn":"2026-10-25","documentId":"arxiv:2607.18966","authors":["Axel Højmark","Jérémy Scheurer 1 1 footnotemark: 1","Evgenia Nitishinskaya","Felix Hofstätter 1 1 footnotemark: 1","Jason Wolfe 2 2 footnotemark: 2","Theodore Ehrenborg 1 1 footnotemark: 1","Bronson Schoen 1 1 footnotemark: 1","Alexander Meinke 1 1 footnotemark: 1","Apollo Research & OpenAI"],"aliases":["https://doi.org/10.48550/arXiv.2607.18966","https://arxiv.org/abs/2607.18966","https://arxiv.org/pdf/2607.18966"]}]}