{"schemaVersion":"nuwa-current-record-1","version":"2026-10-29.1","publication":"published","work":{"id":"mle-bench","title":"MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering","aliases":["MLE-bench"],"authors":["Jun Shern Chan","Neil Chowdhury","Oliver Jaffe","James Aung","Dane Sherburn","Evan Mays","Giulio Starace","Kevin Liu","Leon Maksin","Tejal Patwardhan","Lilian Weng","Aleksander Mądry"],"primaryFunction":"evaluation","secondaryFunctions":[],"kind":"benchmark","sourceId":"mle-bench-publication","attributionSourceId":"mle-bench-publication-author-source","originalUrl":"https://arxiv.org/abs/2410.07095","publicationDate":"2024-10-09","updatedOn":"2026-10-25","areas":["autonomy"],"scopeNote":{"en":"The main dimension follows the task outcome in the original protocol. Topic relevance is documented in the parent research record; no unreviewed question-level links are inferred.","zh":"主维度依据原协议的任务结果确定。主题关联见研究档案，不推断尚未审阅的问题级关联。"},"protocolIds":["mle-bench"],"contributions":[{"organizationId":"openai","role":"research","sourceId":"mle-bench-publication-author-source","locator":"Author affiliations on the original paper title page"}],"summary":{"en":"Evaluates autonomous machine-learning engineering using competition tasks and human performance baselines.","zh":"通过竞赛任务和人类表现基线评测自主机器学习工程能力。"},"note":{"question":null,"method":null,"finding":null,"supports":null,"limits":null,"connections":null},"resources":[],"noteStatus":{"question":"limited","method":"limited","finding":"limited","supports":"limited","limits":"limited","connections":"limited"},"noteAvailability":{"en":"A substantive NUWA research note is not yet available. The repeated draft sections were removed pending review of the original methods, findings and limitations; use the original publication for the research account.","zh":"实质性的 NUWA 研究笔记尚未完成。重复草稿章节已移除，待核查原始方法、发现与限制；研究内容请以原始出版物为准。"},"noteSourceLocator":"No complete section-level method/results/limitations review completed for this note; cached original page and any chart audit retained as discovery evidence.","tags":[]},"reading":{"id":"mle-bench","title":"MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering","en":{"question":"Can agents perform competitive machine-learning engineering with real datasets?","design":"75 curated Kaggle competitions use leaderboard medal thresholds and agent scaffolds, with budget and repeat-attempt experiments.","findings":["o1-preview with AIDE earned at least bronze on 16.9% of competitions at pass@1 and 34.1% at pass@8.","GPT-4o’s reported score rose from 8.7% at 24 hours to 11.8% at 100 hours."],"interpretation":"NUWA interpretation: repeat opportunities and time budget substantially alter competition coverage.","limits":"Historical Kaggle leaderboards are not matched human-time baselines; public solutions create contamination risks.","decisionRelevance":"Model recommendation: always cite medal criterion, scaffold, budget and pass@k together.","takeaway":"NUWA interpretation: repeat opportunities and time budget substantially alter competition coverage.","connections":[],"scoring":"Kaggle medal-level attainment, with pass@1 and pass@k distinguished.","conditions":"75 competitions, scaffold/time/compute and retry conditions.","statistics":"o1-preview with AIDE earned at least bronze on 16.9% of competitions at pass@1 and 34.1% at pass@8. GPT-4o’s reported score rose from 8.7% at 24 hours to 11.8% at 100 hours.","usage":"Model recommendation: always cite medal criterion, scaffold, budget and pass@k together."},"zh":{"question":"智能体能否在真实数据上完成有竞争力的机器学习工程？","design":"75项精选Kaggle竞赛采用排行榜奖牌阈值与智能体框架，并改变预算和重复次数。","findings":["o1-preview配AIDE在pass@1中16.9%获至少铜牌，pass@8为34.1%。","GPT-4o从24小时8.7%升至100小时11.8%。"],"interpretation":"NUWA 解读：重复机会和时间预算显著改变竞赛覆盖。","limits":"历史排行榜不是等时人类基线，公开解答带来污染风险。","decisionRelevance":"模型建议：同时引用奖牌标准、框架、预算与pass@k。","takeaway":"NUWA 解读：重复机会和时间预算显著改变竞赛覆盖。","connections":[],"scoring":"达到Kaggle奖牌级别，区分pass@1与pass@k。","conditions":"75竞赛，保留框架、时间算力及重试条件。","statistics":"o1-preview配AIDE在pass@1中16.9%获至少铜牌，pass@8为34.1%。 GPT-4o从24小时8.7%升至100小时11.8%。","usage":"模型建议：同时引用奖牌标准、框架、预算与pass@k。"},"assertionPolicy":{"question":"source-research-question","design":"source-finding","findings":"source-finding","interpretation":"nuwa-interpretation","limits":"source-limit-or-explicit-editorial-scope-limit","connections":"source-supported-relationship-only","decisionRelevance":"model-recommendation-not-source-conclusion"},"sourceRefs":[{"sourceId":"mle-bench-publication-author-source","url":"https://arxiv.org/html/2410.07095","locator":"Abstract","version":"Selected original resource https://arxiv.org/html/2410.07095; sha256:d6aa28f679c2d75eb0872c3b862f1bc4f27d3ef5efe6859f000f3b41ca98ad02; archived source capture; exact capture time not independently established","artifactSha256":"d6aa28f679c2d75eb0872c3b862f1bc4f27d3ef5efe6859f000f3b41ca98ad02"},{"sourceId":"mle-bench-publication-author-source","url":"https://arxiv.org/html/2410.07095","locator":"§2.1 Dataset Curation","version":"Selected original resource https://arxiv.org/html/2410.07095; sha256:d6aa28f679c2d75eb0872c3b862f1bc4f27d3ef5efe6859f000f3b41ca98ad02; archived source capture; exact capture time not independently established","artifactSha256":"d6aa28f679c2d75eb0872c3b862f1bc4f27d3ef5efe6859f000f3b41ca98ad02"},{"sourceId":"mle-bench-publication-author-source","url":"https://arxiv.org/html/2410.07095","locator":"§1 Introduction: resource-scaling results","version":"Selected original resource https://arxiv.org/html/2410.07095; sha256:d6aa28f679c2d75eb0872c3b862f1bc4f27d3ef5efe6859f000f3b41ca98ad02; archived source capture; exact capture time not independently established","artifactSha256":"d6aa28f679c2d75eb0872c3b862f1bc4f27d3ef5efe6859f000f3b41ca98ad02"},{"sourceId":"mle-bench-publication-author-source","url":"https://arxiv.org/html/2410.07095","locator":"§4 Contamination & Plagiarism","version":"Selected original resource https://arxiv.org/html/2410.07095; sha256:d6aa28f679c2d75eb0872c3b862f1bc4f27d3ef5efe6859f000f3b41ca98ad02; archived source capture; exact capture time not independently established","artifactSha256":"d6aa28f679c2d75eb0872c3b862f1bc4f27d3ef5efe6859f000f3b41ca98ad02"},{"sourceId":"mle-bench-publication-author-source","url":"https://arxiv.org/html/2410.07095","locator":"§6 Limitations","version":"Selected original resource https://arxiv.org/html/2410.07095; sha256:d6aa28f679c2d75eb0872c3b862f1bc4f27d3ef5efe6859f000f3b41ca98ad02; archived source capture; exact capture time not independently established","artifactSha256":"d6aa28f679c2d75eb0872c3b862f1bc4f27d3ef5efe6859f000f3b41ca98ad02"}],"publication":{"state":"published","publishedOn":"2026-10-29","authorizationReference":"user-authorized-global-index-publication-2026-10-29","reviewedCandidateId":"local-6efe31bc4a0c0f37"},"review":{"state":"published-note","sourceScope":"full-text","availableScope":null,"readingCoverage":"selected-source-sections; not a complete section-by-section paper audit","completePaperAudit":false,"independentEvidenceReview":"pending-human-review","bilingualEditorialReview":"pending-human-review","authorship":"model-assisted-research-note","preparedFingerprint":"54169e9837de96cac3fe0c6eecafef6f3fb3be2fe4593baef51008a4b2c3ccd1"}},"sources":[{"id":"mle-bench-publication","title":"MLE-bench: Evaluating Machine Learning Agents on Machine Learning Engineering","url":"https://arxiv.org/abs/2410.07095","version":"arXiv:2410.07095v6","publicationDate":"2024-10-09","datePrecision":"day","type":"paper","verifiedOn":"2026-10-25","documentId":"arxiv:2410.07095","authors":["Chan Jun Shern *","Neil Chowdhury","Oliver Jaffe *","James Aung *","Dane Sherburn *","Evan Mays *","Giulio Starace *","Kevin Liu","Leon Maksin","Tejal Patwardhan","Lilian Weng","Aleksander MądryOpenAI"],"aliases":["https://doi.org/10.48550/arXiv.2410.07095","https://arxiv.org/abs/2410.07095","https://arxiv.org/pdf/2410.07095"]}]}